00010000:00000010:0.1F:1787826376.848002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00010000:00000010:1.1F:1787826385.009385:0:14641:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97f00.
00010000:00000010:1.1:1787826385.009412:0:14641:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:1.1:1787826385.027795:0:14641:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85eacde0.
00010000:00000010:1.1:1787826385.088692:0:14652:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:1.1:1787826385.088709:0:14652:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f58a80.
00010000:00000010:0.1:1787826385.111690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826385.111714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85eac920.
00010000:00000010:3.1F:1787826385.150730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00010000:00000010:3.1:1787826385.152719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826385.243903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84dd2a00.
00010000:00000010:1.1:1787826385.250795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826385.250814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:0.1:1787826385.283739:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826385.283760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85eac040.
00010000:00000010:3.1:1787826385.319714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:3.1:1787826385.319734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f58de0.
00010000:00000010:3.1:1787826385.344353:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:2.1F:1787826385.350728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f582c0.
00010000:00000010:0.1:1787826385.397706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00010000:00000010:0.1:1787826385.397715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:3.1:1787826385.473231:0:14657:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87f70ce0.
00010000:00000010:0.1:1787826385.522725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826385.522745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:0.1:1787826385.536132:0:14658:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:0.1:1787826385.536143:0:14658:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f58e60.
00010000:00000010:0.1:1787826385.552963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826385.560728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84dd2980.
00010000:00000010:1.1:1787826385.572272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:1.1:1787826385.572282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84dd2c60.
00010000:00000010:3.1:1787826385.604869:0:14658:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00010000:00000010:3.1:1787826385.604877:0:14658:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826385.729780:0:14640:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87f70440.
00010000:00000010:1.1:1787826385.776721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826385.776742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:1.1:1787826385.776753:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84506b40.
00000080:00000010:1.1:1787826385.776762:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87f70dc0.
00010000:00000010:0.1:1787826385.828136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:0.1:1787826385.828159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85fda780.
00010000:00000010:0.1:1787826385.835735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826385.835757:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d714a0.
00010000:00000010:1.1:1787826385.876459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00010000:00000010:1.1:1787826385.876473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:2.1:1787826385.944772:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85fda4e0.
00010000:00000010:2.1:1787826385.970473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43a00.
00010000:00000010:2.1:1787826385.970492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:2.1:1787826385.986632:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84518840.
00000080:00000010:2.1:1787826385.986677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9012e580.
00010000:00000010:2.1:1787826386.028306:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826386.028318:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87fc55a0.
00010000:00000010:2.1:1787826386.028325:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826386.028327:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87fc5480.
00010000:00000010:1.1:1787826386.059732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:3.1:1787826386.099992:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d71660.
00010000:00000010:3.1:1787826386.133983:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97e00.
00010000:00000010:3.1:1787826386.134002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:3.1:1787826386.160444:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:3.1:1787826386.160457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d71c60.
00010000:00000010:0.1:1787826386.180535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:0.1:1787826386.180543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87fc5100.
00010000:00000010:0.1:1787826386.188487:0:14676:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826386.188505:0:14676:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d71040.
00010000:00000010:1.1:1787826386.212770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826386.264569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87fc5a00.
00010000:00000010:1.1:1787826386.289051:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826386.289072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826386.302713:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84512940.
00000080:00000010:1.1:1787826386.302736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85fdabc0.
00010000:00000010:1.1:1787826386.355215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826386.355238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85fdac00.
00010000:00000010:1.1:1787826386.368009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826386.368032:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85fda780.
00010000:00000010:2.1:1787826386.384883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826386.444284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d71da0.
00010000:00000010:3.1:1787826386.463114:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97e00.
00010000:00000010:3.1:1787826386.463131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826386.490294:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:3.1:1787826386.490316:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd841bdd60.
00010000:00000010:1.1:1787826386.522922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:1.1:1787826386.522946:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86127cc0.
00010000:00000010:1.1:1787826386.543930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826386.543951:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85e10480.
00010000:00000010:0.1:1787826386.573721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:2.1:1787826386.602720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f58140.
00010000:00000010:1.1:1787826386.635842:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826386.635861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:1.1:1787826386.647304:0:14675:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84511080.
00000080:00000010:1.1:1787826386.647343:0:14675:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861279a0.
00010000:00000010:2.1:1787826386.699723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826386.699744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd886e4c40.
00010000:00000010:2.1:1787826386.711862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826386.711883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd886e4960.
00010000:00000010:2.1:1787826386.739857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826386.836452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f31940.
00010000:00000010:0.1:1787826386.840709:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826386.840732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:0.1:1787826386.860798:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84518840.
00000080:00000010:0.1:1787826386.860831:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86127480.
00010000:00000010:2.1:1787826386.890717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:2.1:1787826386.890738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f319c0.
00000080:00000010:3.1:1787826386.930085:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86127ba0.
00010000:00000010:2.1:1787826386.932799:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00010000:00000010:0.1:1787826386.941733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:1.1:1787826386.992580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85e10f80.
00010000:00000010:2.1:1787826387.025853:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43a00.
00010000:00000010:2.1:1787826387.025874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826387.053695:0:14691:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84511080.
00000080:00000010:2.1:1787826387.053717:0:14691:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f31fe0.
00010000:00000010:1.1:1787826387.105630:0:14675:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:1.1:1787826387.105646:0:14675:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861273a0.
00010000:00000010:0.1:1787826387.109173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:0.1:1787826387.116688:0:14691:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85e10b00.
00010000:00000010:0.1:1787826387.135589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826387.206707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f315c0.
00010000:00000010:1.1:1787826387.255134:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826387.255144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826387.255155:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826387.255166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85e10f60.
00010000:00000010:1.1:1787826387.280698:0:14699:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:1.1:1787826387.280705:0:14699:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f313e0.
00010000:00000010:3.1:1787826387.284442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:3.1:1787826387.284462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85e10820.
00010000:00000010:2.1:1787826387.293736:0:14703:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:1.1:1787826387.351395:0:14640:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d71720.
00010000:00000010:0.1:1787826387.379584:0:14706:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826387.379855:0:14706:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:0.1:1787826387.392810:0:14707:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84512940.
00000080:00000010:0.1:1787826387.392823:0:14707:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86127a20.
00010000:00000010:1.1:1787826387.414776:0:14707:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826387.414785:0:14707:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85e10a60.
00010000:00000010:1.1:1787826387.414795:0:14707:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826387.414799:0:14707:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd82c460a0.
00010000:00000010:1.1:1787826387.426729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:0.1:1787826387.451728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86127a20.
00010000:00000010:1.1:1787826387.470759:0:14709:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826387.470770:0:14709:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:1.1:1787826387.479737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84526b40.
00000080:00000010:1.1:1787826387.479759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861279a0.
00010000:00000010:0.1:1787826387.519699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826387.519741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8435a9a0.
00000080:00000010:2.1:1787826387.527794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d50440.
00010000:00000010:1.1:1787826387.535052:0:14710:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00010000:00000010:1.1:1787826387.559710:0:14711:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:1.1:1787826387.603724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d50ea0.
00010000:00000010:0.1:1787826387.614730:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826387.614752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:0.1:1787826387.648186:0:14713:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84526b40.
00000080:00000010:0.1:1787826387.648215:0:14713:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b6c0.
00010000:00000010:0.1:1787826387.688725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:0.1:1787826387.688743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd91c04360.
00010000:00000010:2.1:1787826387.689031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:2.1:1787826387.689038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84378300.
00010000:00000010:2.1:1787826387.695900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:0.1:1787826387.738707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84378140.
00010000:00000010:2.1:1787826387.779784:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43a00.
00010000:00000010:2.1:1787826387.779809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:2.1:1787826387.785865:0:14716:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451b9c0.
00000080:00000010:2.1:1787826387.785878:0:14716:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b6a0.
00010000:00000010:0.1:1787826387.822919:0:14716:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:0.1:1787826387.822928:0:14716:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b9c0.
00010000:00000010:0.1:1787826387.829836:0:14716:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826387.829846:0:14716:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84378fe0.
00010000:00000010:0.1:1787826387.841383:0:14716:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:2.1:1787826387.916794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b180.
00010000:00000010:1.1:1787826387.931766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826387.931788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:0.1:1787826387.960710:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84512940.
00000080:00000010:0.1:1787826387.960732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd91c042a0.
00010000:00000010:0.1:1787826387.990204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826387.996696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b3c0.
00010000:00000010:0.1:1787826388.000722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826388.000744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84378780.
00010000:00000010:0.1:1787826388.032704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826388.083648:0:14722:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd843786e0.
00010000:00000010:0.1:1787826388.125489:0:12446:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826388.125508:0:12446:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:2.1:1787826388.127701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d501c0.
00010000:00000010:0.1:1787826388.200698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826388.200718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd926020a0.
00010000:00000010:0.1:1787826388.217091:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826388.221095:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd926029c0.
00010000:00000010:0.1:1787826388.248787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826388.331872:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430bb60.
00010000:00000010:1.1:1787826388.345846:0:14731:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826388.345861:0:14731:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826388.358740:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84505ac0.
00000080:00000010:1.1:1787826388.358761:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d502c0.
00010000:00000010:3.1:1787826388.380733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826388.380756:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b1e0.
00010000:00000010:3.1:1787826388.403941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:0.1:1787826388.404476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b240.
00010000:00000010:0.1:1787826388.420352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826388.456698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430bcc0.
00010000:00000010:1.1:1787826388.489872:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826388.489894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826388.516721:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84512940.
00000080:00000010:1.1:1787826388.516732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d50e20.
00010000:00000010:0.1:1787826388.528363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826388.533737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92602ce0.
00010000:00000010:0.1:1787826388.543520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826388.548153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92602380.
00010000:00000010:1.1:1787826388.558730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:3.1:1787826388.594264:0:14640:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd91c04520.
00010000:00000010:0.1:1787826388.647791:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826388.647813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:1.1:1787826388.673038:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:1.1:1787826388.673061:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84378720.
00000080:00000010:3.1:1787826388.725721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd83699080.
00010000:00000010:2.1:1787826388.725819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:1.1:1787826388.729530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8430b800.
00010000:00000010:0.1:1787826388.729695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00010000:00000010:1.1:1787826388.751723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826388.800456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9258b9e0.
00010000:00000010:3.1:1787826388.821771:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97500.
00010000:00000010:3.1:1787826388.821786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:3.1:1787826388.833725:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:3.1:1787826388.833747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9258b300.
00010000:00000010:0.1:1787826388.874746:0:14743:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826388.874752:0:14743:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd83699f60.
00010000:00000010:0.1:1787826388.881711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826388.881723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9258b400.
00010000:00000010:0.1:1787826388.902100:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:3.1:1787826388.946505:0:14745:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85ee7c80.
00010000:00000010:0.1:1787826388.973579:0:14745:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826388.973637:0:14745:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826388.984722:0:14746:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826388.984743:0:14746:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9258b580.
00010000:00000010:3.1:1787826389.030682:0:14746:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:3.1:1787826389.030760:0:14746:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd83699d00.
00010000:00000010:3.1:1787826389.036725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:3.1:1787826389.036748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd83699920.
00010000:00000010:1.1:1787826389.066292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:1.1:1787826389.205514:0:14753:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84cb4060.
00010000:00000010:1.1:1787826389.225038:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826389.225053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826389.249753:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84521080.
00000080:00000010:1.1:1787826389.249844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84cb4520.
00010000:00000010:0.1:1787826389.305350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826389.305369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84cb47e0.
00010000:00000010:0.1:1787826389.340432:0:14758:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826389.340444:0:14758:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9258ba40.
00010000:00000010:1.1:1787826389.360732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:2.1:1787826389.414814:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f157e0.
00010000:00000010:1.1:1787826389.434877:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826389.434896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826389.460150:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451b9c0.
00000080:00000010:1.1:1787826389.460168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84cb4d80.
00000080:00000010:0.1:1787826389.472812:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9012f8a0.
00010000:00000010:1.1:1787826389.477362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00010000:00000010:0.1:1787826389.483435:0:14761:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826389.483445:0:14761:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85d33a20.
00010000:00000010:0.1:1787826389.505741:0:14757:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:2.1:1787826389.536101:0:14763:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f158c0.
00010000:00000010:1.1:1787826389.557024:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826389.557044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:1.1:1787826389.574843:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:1.1:1787826389.574869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85ee7600.
00010000:00000010:0.1:1787826389.617039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826389.617060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85d33b40.
00010000:00000010:0.1:1787826389.651713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826389.651731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85d33420.
00010000:00000010:3.1:1787826389.663511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:2.1:1787826389.708743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f156e0.
00000080:00000010:3.1:1787826389.764067:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84502940.
00000080:00000010:3.1:1787826389.764089:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9012f3e0.
00010000:00000010:1.1:1787826389.764245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707900.
00010000:00000010:1.1:1787826389.764255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00010000:00000010:1.1:1787826389.806096:0:14767:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826389.811147:0:14767:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9012ff60.
00010000:00000010:1.1:1787826389.814702:0:14767:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826389.814714:0:14767:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9012fda0.
00010000:00000010:1.1:1787826389.823726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:3.1:1787826389.882703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85ee7520.
00010000:00000010:1.1:1787826389.912940:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707900.
00010000:00000010:1.1:1787826389.912961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826389.918769:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84521080.
00000080:00000010:1.1:1787826389.918792:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87426860.
00010000:00000010:1.1:1787826389.944774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826389.944785:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85d33f40.
00010000:00000010:1.1:1787826389.956007:0:14770:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826389.956089:0:14770:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87426d40.
00010000:00000010:2.1:1787826389.967936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:3.1:1787826390.009698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85ee7740.
00010000:00000010:0.1:1787826390.029343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826390.031057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:1.1:1787826390.038617:0:14773:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:1.1:1787826390.038638:0:14773:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85d33700.
00010000:00000010:2.1:1787826390.089065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:2.1:1787826390.089082:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f15fe0.
00010000:00000010:0.1:1787826390.096722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:0.1:1787826390.096732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f15000.
00010000:00000010:0.1:1787826390.114126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:2.1:1787826390.147048:0:14775:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f156e0.
00010000:00000010:0.1:1787826390.171179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826390.171216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:0.1:1787826390.185840:0:14776:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514a40.
00000080:00000010:0.1:1787826390.185864:0:14776:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f15f60.
00010000:00000010:0.1:1787826390.219226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:2.1:1787826390.224111:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd82c9eca0.
00010000:00000010:0.1:1787826390.228711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:0.1:1787826390.228724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f15700.
00010000:00000010:1.1:1787826390.258886:0:14777:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826390.313329:0:14778:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d07620.
00010000:00000010:1.1:1787826390.362919:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707900.
00010000:00000010:1.1:1787826390.362935:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:0.1:1787826390.363701:0:14782:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87426f80.
00010000:00000010:0.1:1787826390.431104:0:14787:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:0.1:1787826390.431120:0:14787:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d077c0.
00010000:00000010:3.1:1787826390.435345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:3.1:1787826390.435363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:1.1:1787826390.476634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:2.1:1787826390.517721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d070c0.
00010000:00000010:0.1:1787826390.559829:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707900.
00010000:00000010:0.1:1787826390.559852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:0.1:1787826390.588853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520000.
00000080:00000010:0.1:1787826390.588871:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:1.1:1787826390.620903:0:14791:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826390.620912:0:14791:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fc60.
00010000:00000010:1.1:1787826390.629087:0:14791:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826390.629097:0:14791:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f980.
00010000:00000010:1.1:1787826390.639916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826390.700096:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969120.
00010000:00000010:2.1:1787826390.720724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43a00.
00010000:00000010:2.1:1787826390.720746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826390.729905:0:1032:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84527380.
00000080:00000010:2.1:1787826390.729925:0:1032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d07c20.
00010000:00000010:3.1:1787826390.802230:0:14794:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826390.809160:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd871a9f40.
00000080:00000010:2.1:1787826390.813166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd871a9920.
00010000:00000010:3.1:1787826390.813359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00010000:00000010:2.1:1787826390.837520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:0.1:1787826390.907424:0:14796:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826390.957717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:1.1:1787826390.957738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:3.1:1787826390.980716:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514a40.
00000080:00000010:3.1:1787826390.980739:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969aa0.
00010000:00000010:0.1:1787826391.029705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:1.1:1787826391.029719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d07340.
00010000:00000010:1.1:1787826391.046738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826391.051715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9e0.
00010000:00000010:2.1:1787826391.068483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826391.109312:0:14799:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:0.1:1787826391.127849:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826391.127871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826391.159728:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845139c0.
00000080:00000010:0.1:1787826391.159751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:1.1:1787826391.200712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826391.208748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:0.1:1787826391.230696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826391.230973:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:1.1:1787826391.235725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:2.1:1787826391.314914:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969460.
00010000:00000010:0.1:1787826391.372962:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826391.372983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:3.1:1787826391.417226:0:14803:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f400.
00010000:00000010:3.1:1787826391.435911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:3.1:1787826391.435927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd871a94e0.
00010000:00000010:3.1:1787826391.449923:0:14803:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826391.449930:0:14803:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd871a9140.
00010000:00000010:2.1:1787826391.476899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:1.1:1787826391.514247:0:14808:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969660.
00010000:00000010:1.1:1787826391.543744:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826391.543766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:1.1:1787826391.568734:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84502940.
00000080:00000010:1.1:1787826391.568743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd871a9d60.
00010000:00000010:1.1:1787826391.598823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826391.598844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826391.617763:0:14815:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826391.617774:0:14815:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:0.1:1787826391.621827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:3.1:1787826391.697247:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134640.
00010000:00000010:2.1:1787826391.700713:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:2.1:1787826391.700736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826391.711109:0:14818:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84510840.
00000080:00000010:0.1:1787826391.711130:0:14818:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f540.
00010000:00000010:0.1:1787826391.745410:0:14:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826391.745422:0:14:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929692a0.
00010000:00000010:0.1:1787826391.751813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826391.751828:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:2.1:1787826391.757713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:2.1:1787826391.810728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826391.832993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:1.1:1787826391.833025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826391.865296:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451a100.
00000080:00000010:1.1:1787826391.865317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:2.1:1787826391.882432:0:14821:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826391.882446:0:14821:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87efe980.
00010000:00000010:2.1:1787826391.893848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826391.893857:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87efeee0.
00010000:00000010:1.1:1787826391.910737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:3.1:1787826391.966992:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134600.
00010000:00000010:2.1:1787826391.982135:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43200.
00010000:00000010:2.1:1787826391.982158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826391.993076:0:14824:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:2.1:1787826391.993105:0:14824:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87efe680.
00010000:00000010:1.1:1787826392.053852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:3.1:1787826392.056867:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fce0.
00010000:00000010:0.1:1787826392.070865:0:14824:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:0.1:1787826392.070876:0:14824:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134ea0.
00010000:00000010:0.1:1787826392.084025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826392.111972:0:14826:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:0.1:1787826392.142738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:0.1:1787826392.142765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826392.158693:0:14814:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84510840.
00000080:00000010:0.1:1787826392.158716:0:14814:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:3.1:1787826392.199354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:3.1:1787826392.199377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134560.
00000080:00000010:2.1:1787826392.211925:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861349a0.
00010000:00000010:3.1:1787826392.212089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00010000:00000010:0.1:1787826392.235742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:1.1:1787826392.301704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134bc0.
00010000:00000010:2.1:1787826392.320765:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43200.
00010000:00000010:2.1:1787826392.320792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:1.1:1787826392.328563:0:14830:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84510840.
00000080:00000010:1.1:1787826392.328591:0:14830:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87efe380.
00010000:00000010:2.1:1787826392.376713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:2.1:1787826392.386710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7c0.
00010000:00000010:1.1:1787826392.394513:0:14812:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:1.1:1787826392.394542:0:14812:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2520.
00010000:00000010:0.1:1787826392.398932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826392.490769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:3.1:1787826392.518077:0:14842:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97500.
00010000:00000010:3.1:1787826392.518094:0:14842:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:1.1:1787826392.540714:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84516300.
00000080:00000010:1.1:1787826392.540734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861340e0.
00010000:00000010:0.1:1787826392.581720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:0.1:1787826392.581742:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861349a0.
00010000:00000010:0.1:1787826392.608144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:2.1:1787826392.618110:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:1.1:1787826392.632007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:0.1:1787826392.686763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929691e0.
00010000:00000010:3.1:1787826392.707006:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97500.
00010000:00000010:3.1:1787826392.707028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:3.1:1787826392.732728:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84524200.
00000080:00000010:3.1:1787826392.732747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:0.1:1787826392.759920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:0.1:1787826392.759941:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:0.1:1787826392.767718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826392.778979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969960.
00010000:00000010:1.1:1787826392.800723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:2.1:1787826392.842704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a80.
00010000:00000010:2.1:1787826392.869837:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43200.
00010000:00000010:2.1:1787826392.869856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:0.1:1787826392.886886:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84526300.
00000080:00000010:0.1:1787826392.886908:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:1.1:1787826392.920680:0:14849:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826392.920852:0:14849:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fec0.
00010000:00000010:1.1:1787826392.930889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826392.940737:0:14849:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:0.1:1787826392.953904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:2.1:1787826393.018689:0:14851:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969de0.
00010000:00000010:2.1:1787826393.057916:0:2341:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707200.
00010000:00000010:2.1:1787826393.057936:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:0.1:1787826393.064121:0:14852:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826393.064144:0:14852:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2320.
00010000:00000010:0.1:1787826393.101404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826393.101419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:0.1:1787826393.101431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826393.101434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929692c0.
00010000:00000010:2.1:1787826393.106306:0:14852:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:2.1:1787826393.143131:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2880.
00010000:00000010:2.1:1787826393.188764:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda700.
00010000:00000010:2.1:1787826393.188784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:2.1:1787826393.222773:0:14855:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451ca40.
00000080:00000010:2.1:1787826393.222784:0:14855:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695a0.
00010000:00000010:2.1:1787826393.259878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:2.1:1787826393.259896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2820.
00010000:00000010:2.1:1787826393.268218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826393.268238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2f80.
00010000:00000010:2.1:1787826393.282612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:1.1:1787826393.321021:0:14857:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826393.354112:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826393.354128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:2.1:1787826393.362691:0:14858:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:2.1:1787826393.362711:0:14858:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:3.1:1787826393.406272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:3.1:1787826393.406287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702afa0.
00010000:00000010:3.1:1787826393.410718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:3.1:1787826393.410790:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702a740.
00010000:00000010:1.1:1787826393.433719:0:14860:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:2.1:1787826393.482726:0:14869:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:0.1:1787826393.500321:0:14869:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826393.500340:0:14869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826393.517172:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451ca40.
00000080:00000010:0.1:1787826393.517202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:2.1:1787826393.558749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:2.1:1787826393.558765:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702af40.
00010000:00000010:3.1:1787826393.586256:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:3.1:1787826393.586277:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:1.1:1787826393.625124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:1.1:1787826393.650719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:3.1:1787826393.680736:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43200.
00010000:00000010:3.1:1787826393.680758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826393.687757:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520840.
00000080:00000010:0.1:1787826393.687865:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:1.1:1787826393.727306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826393.727328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7c0.
00010000:00000010:3.1:1787826393.749064:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:3.1:1787826393.749083:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:2.1:1787826393.762886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:0.1:1787826393.806022:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969460.
00010000:00000010:0.1:1787826393.830649:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826393.830674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826393.862797:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84513180.
00000080:00000010:0.1:1787826393.862818:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969460.
00010000:00000010:3.1:1787826393.881120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:3.1:1787826393.881129:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f540.
00010000:00000010:0.1:1787826393.886997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:0.1:1787826393.890166:0:14876:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459fc0.
00010000:00000010:0.1:1787826393.901463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:1.1:1787826393.961721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9e0.
00010000:00000010:2.1:1787826393.970770:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43f00.
00010000:00000010:2.1:1787826393.970789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:0.1:1787826394.006713:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84513180.
00000080:00000010:0.1:1787826394.006738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:0.1:1787826394.037829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:0.1:1787826394.037850:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26e0.
00000080:00000010:1.1:1787826394.043048:0:14879:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969460.
00010000:00000010:0.1:1787826394.043886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00010000:00000010:1.1:1787826394.047478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826394.092499:0:14881:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459020.
00010000:00000010:0.1:1787826394.136118:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826394.136136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826394.140088:0:14882:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520840.
00000080:00000010:0.1:1787826394.140108:0:14882:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f400.
00000080:00000010:1.1:1787826394.165802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:0.1:1787826394.169705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00010000:00000010:1.1:1787826394.175798:0:14882:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826394.175814:0:14882:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9e0.
00010000:00000010:1.1:1787826394.196731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826394.234718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969aa0.
00010000:00000010:1.1:1787826394.287105:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:1.1:1787826394.287123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:2.1:1787826394.307755:0:14864:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599a0.
00010000:00000010:2.1:1787826394.327810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:2.1:1787826394.327825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ea0.
00010000:00000010:1.1:1787826394.330962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826394.332038:0:14885:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c80.
00010000:00000010:3.1:1787826394.349446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:3.1:1787826394.414268:0:14892:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ca0.
00010000:00000010:0.1:1787826394.445690:0:14896:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826394.445704:0:14896:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:3.1:1787826394.465208:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84505280.
00000080:00000010:3.1:1787826394.465234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874596c0.
00010000:00000010:1.1:1787826394.510719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:0.1:1787826394.514332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e23c0.
00010000:00000010:0.1:1787826394.530549:0:14897:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826394.530562:0:14897:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969cc0.
00010000:00000010:0.1:1787826394.546797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826394.595737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f400.
00010000:00000010:0.1:1787826394.614125:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826394.614134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:0.1:1787826394.649738:0:14900:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84505280.
00000080:00000010:0.1:1787826394.649759:0:14900:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26e0.
00010000:00000010:1.1:1787826394.671710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826394.671732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f540.
00010000:00000010:1.1:1787826394.680471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826394.680491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:1.1:1787826394.725904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826394.749871:0:14902:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:1.1:1787826394.768109:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:1.1:1787826394.768118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826394.785288:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84524200.
00000080:00000010:1.1:1787826394.785314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fc40.
00000080:00000010:1.1:1787826394.811280:0:14896:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:0.1:1787826394.811714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00010000:00000010:0.1:1787826394.821955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826394.821977:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fc40.
00010000:00000010:0.1:1787826394.833907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826394.884738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969920.
00010000:00000010:3.1:1787826394.920954:0:14906:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97800.
00010000:00000010:3.1:1787826394.920970:0:14906:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:3.1:1787826394.921052:0:14906:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84513180.
00000080:00000010:3.1:1787826394.921065:0:14906:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f4e0.
00010000:00000010:0.1:1787826394.964142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826394.964156:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969920.
00010000:00000010:1.1:1787826394.980280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826394.980700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:1.1:1787826394.987226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826395.064869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969920.
00010000:00000010:1.1:1787826395.083588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:1.1:1787826395.083610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826395.099723:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84524200.
00000080:00000010:1.1:1787826395.099743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:0.1:1787826395.134723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826395.134730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969aa0.
00010000:00000010:0.1:1787826395.141818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826395.141833:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:2.1:1787826395.153861:0:14910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:2.1:1787826395.178777:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22e0.
00010000:00000010:1.1:1787826395.204940:0:14896:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:1.1:1787826395.204959:0:14896:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826395.209095:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84501080.
00000080:00000010:1.1:1787826395.209114:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826395.237022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826395.237031:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695e0.
00010000:00000010:0.1:1787826395.245279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826395.245297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969620.
00010000:00000010:3.1:1787826395.252613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:3.1:1787826395.300722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:0.1:1787826395.314677:0:14914:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826395.314685:0:14914:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826395.322717:0:14915:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84500000.
00000080:00000010:0.1:1787826395.322735:0:14915:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969660.
00010000:00000010:2.1:1787826395.367331:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:2.1:1787826395.367343:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:3.1:1787826395.380449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:3.1:1787826395.380473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874590e0.
00010000:00000010:1.1:1787826395.407693:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:2.1:1787826395.500256:0:14925:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:0.1:1787826395.515722:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda000.
00010000:00000010:0.1:1787826395.515745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:0.1:1787826395.541743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845198c0.
00000080:00000010:0.1:1787826395.541769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874590e0.
00010000:00000010:0.1:1787826395.588450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826395.588469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ae0.
00010000:00000010:0.1:1787826395.606790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826395.606814:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690a0.
00010000:00000010:2.1:1787826395.622713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:1.1:1787826395.653752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459bc0.
00010000:00000010:1.1:1787826395.701342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:1.1:1787826395.701359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:1.1:1787826395.711449:0:14930:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84507380.
00000080:00000010:1.1:1787826395.711480:0:14930:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459660.
00010000:00000010:3.1:1787826395.759518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:3.1:1787826395.779750:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00010000:00000010:3.1:1787826395.783701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826395.783713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459920.
00010000:00000010:2.1:1787826395.806978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:1.1:1787826395.847713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f280.
00010000:00000010:3.1:1787826395.872427:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97800.
00010000:00000010:3.1:1787826395.872447:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:2.1:1787826395.888902:0:14933:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84524200.
00000080:00000010:2.1:1787826395.888925:0:14933:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:3.1:1787826395.927727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:3.1:1787826395.927746:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:0.1:1787826395.932650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:1.1:1787826395.938843:0:14933:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:1.1:1787826395.966724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:2.1:1787826395.999862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:1.1:1787826396.024045:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:1.1:1787826396.024076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:1.1:1787826396.044722:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826396.044745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26c0.
00010000:00000010:0.1:1787826396.079712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:1.1:1787826396.083456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929691e0.
00010000:00000010:1.1:1787826396.091723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826396.091746:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826396.095771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826396.151716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:2.1:1787826396.175111:0:2341:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda600.
00010000:00000010:2.1:1787826396.175135:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:0.1:1787826396.193802:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:0.1:1787826396.193817:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00000080:00000010:1.1:1787826396.211456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ac0.
00010000:00000010:0.1:1787826396.213990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00010000:00000010:1.1:1787826396.218563:0:14939:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826396.218574:0:14939:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826396.230761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826396.277888:0:14941:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:0.1:1787826396.298892:0:14941:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826396.298901:0:14941:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826396.310594:0:14943:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:3.1:1787826396.351840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:3.1:1787826396.351849:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:3.1:1787826396.361709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:3.1:1787826396.361723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:1.1:1787826396.388055:0:14951:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:2.1:1787826396.427976:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:2.1:1787826396.440258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97800.
00010000:00000010:2.1:1787826396.440273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:0.1:1787826396.456226:0:14954:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84503180.
00000080:00000010:0.1:1787826396.456247:0:14954:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:0.1:1787826396.484866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:0.1:1787826396.484883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:0.1:1787826396.489703:0:14954:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826396.489712:0:14954:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969620.
00010000:00000010:0.1:1787826396.502999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:1.1:1787826396.543777:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fce0.
00010000:00000010:0.1:1787826396.566713:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826396.566735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826396.568773:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520000.
00000080:00000010:0.1:1787826396.568791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00000080:00000010:1.1:1787826396.589847:0:14957:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:0.1:1787826396.597706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00010000:00000010:1.1:1787826396.597795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826396.603870:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f940.
00010000:00000010:1.1:1787826396.603895:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826396.637552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:0.1:1787826396.667877:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826396.667896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:1.1:1787826396.678218:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:1.1:1787826396.678233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c60.
00010000:00000010:2.1:1787826396.707871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:2.1:1787826396.707881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:1.1:1787826396.712715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826396.712727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2960.
00010000:00000010:2.1:1787826396.745702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:1.1:1787826396.790713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f4e0.
00010000:00000010:0.1:1787826396.823025:0:14962:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826396.823048:0:14962:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:1.1:1787826396.832354:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:1.1:1787826396.832372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:3.1:1787826396.862756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:3.1:1787826396.862769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459700.
00010000:00000010:3.1:1787826396.877722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:3.1:1787826396.877733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599c0.
00010000:00000010:1.1:1787826396.895058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:2.1:1787826396.949710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00010000:00000010:1.1:1787826396.961816:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707800.
00010000:00000010:1.1:1787826396.961833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826396.981714:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84504200.
00000080:00000010:1.1:1787826396.987715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:2.1:1787826397.005583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:2.1:1787826397.005608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:2.1:1787826397.013728:0:14966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:2.1:1787826397.013745:0:14966:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2300.
00010000:00000010:3.1:1787826397.034629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:2.1:1787826397.091893:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d20.
00010000:00000010:0.1:1787826397.128589:0:14972:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826397.128606:0:14972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:0.1:1787826397.164094:0:14974:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459100.
00010000:00000010:1.1:1787826397.179714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:2.1:1787826397.188700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:2.1:1787826397.223010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826397.223025:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:2.1:1787826397.230464:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826397.318783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f0a0.
00010000:00000010:0.1:1787826397.340730:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826397.340745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:2.1:1787826397.358956:0:14981:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:2.1:1787826397.358981:0:14981:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:0.1:1787826397.398931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:3.1:1787826397.404565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c80.
00010000:00000010:0.1:1787826397.412308:0:14981:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:0.1:1787826397.412326:0:14981:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:0.1:1787826397.427734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826397.483627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f0a0.
00010000:00000010:2.1:1787826397.519025:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707a00.
00010000:00000010:2.1:1787826397.519048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826397.527019:0:14984:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451eb40.
00000080:00000010:0.1:1787826397.527041:0:14984:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:1.1:1787826397.559145:0:14984:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826397.559155:0:14984:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f0a0.
00010000:00000010:1.1:1787826397.559162:0:14984:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826397.559165:0:14984:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:1.1:1787826397.571920:0:14984:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826397.612711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:1.1:1787826397.638721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707200.
00010000:00000010:1.1:1787826397.638742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826397.644638:0:14987:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:0.1:1787826397.644670:0:14987:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f540.
00010000:00000010:1.1:1787826397.666900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:2.1:1787826397.666970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f4e0.
00010000:00000010:2.1:1787826397.678705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826397.678740:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:3.1:1787826397.691277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:0.1:1787826397.732099:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874594c0.
00010000:00000010:2.1:1787826397.754935:0:14989:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43f00.
00010000:00000010:2.1:1787826397.754945:0:14989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826397.766782:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:2.1:1787826397.766804:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00000080:00000010:1.1:1787826397.791865:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:0.1:1787826397.800729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00010000:00000010:1.1:1787826397.804846:0:14990:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826397.811713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826397.814940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826397.894552:0:14992:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:1.1:1787826397.939714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707200.
00010000:00000010:1.1:1787826397.939733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:2.1:1787826397.957819:0:14995:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:2.1:1787826397.999548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:2.1:1787826397.999556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2fe0.
00010000:00000010:1.1:1787826398.004801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:1.1:1787826398.004811:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26c0.
00010000:00000010:1.1:1787826398.015916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:2.1:1787826398.075722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:0.1:1787826398.110173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826398.110244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826398.153073:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:0.1:1787826398.153095:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:0.1:1787826398.171705:0:15005:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:0.1:1787826398.171716:0:15005:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:0.1:1787826398.189343:0:15005:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826398.189355:0:15005:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:1.1:1787826398.205778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826398.269794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:0.1:1787826398.299400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826398.299409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826398.326622:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84513180.
00000080:00000010:0.1:1787826398.326633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969120.
00010000:00000010:3.1:1787826398.342961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:3.1:1787826398.346261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2940.
00010000:00000010:0.1:1787826398.376680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:0.1:1787826398.376700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874590e0.
00010000:00000010:1.1:1787826398.406028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:2.1:1787826398.471862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2040.
00010000:00000010:0.1:1787826398.498725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826398.498750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:0.1:1787826398.527597:0:15011:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826398.527619:0:15011:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c20.
00010000:00000010:0.1:1787826398.561230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826398.561241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:0.1:1787826398.561251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826398.561254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:0.1:1787826398.573981:0:15011:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826398.614742:0:15013:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874592e0.
00010000:00000010:0.1:1787826398.641500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826398.641521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826398.684199:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84513180.
00000080:00000010:1.1:1787826398.684212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:0.1:1787826398.730031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:0.1:1787826398.730041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c00.
00010000:00000010:1.1:1787826398.747931:0:15014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826398.747942:0:15014:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826398.773959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:3.1:1787826398.836630:0:15016:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874598a0.
00010000:00000010:0.1:1787826398.885840:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826398.885851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:0.1:1787826398.899437:0:14997:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826398.899456:0:14997:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:1.1:1787826398.938683:0:15017:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826398.938689:0:15017:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2e0.
00010000:00000010:1.1:1787826398.949876:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826398.949893:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:1.1:1787826398.968743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:3.1:1787826399.022698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ffe0.
00010000:00000010:2.1:1787826399.052304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97e00.
00010000:00000010:2.1:1787826399.052321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:0.1:1787826399.073732:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84506300.
00000080:00000010:0.1:1787826399.073750:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2fc0.
00010000:00000010:1.1:1787826399.099381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826399.101750:0:15029:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826399.112707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826399.112716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f280.
00010000:00000010:0.1:1787826399.116907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826399.168800:0:15031:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2820.
00010000:00000010:1.1:1787826399.197244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707200.
00010000:00000010:1.1:1787826399.197254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826399.215732:0:1343:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826399.215747:0:1343:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:3.1:1787826399.244175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:3.1:1787826399.244190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:0.1:1787826399.269614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:0.1:1787826399.269627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ee0.
00010000:00000010:0.1:1787826399.273880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826399.310310:0:15034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c80.
00010000:00000010:1.1:1787826399.328722:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707200.
00010000:00000010:1.1:1787826399.328736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826399.361305:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84516300.
00000080:00000010:1.1:1787826399.361316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:1.1:1787826399.376941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826399.376958:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fbc0.
00010000:00000010:1.1:1787826399.385203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826399.385227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:1.1:1787826399.389715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826399.439925:0:15037:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:0.1:1787826399.453782:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826399.453808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826399.468960:0:15038:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845239c0.
00000080:00000010:1.1:1787826399.468973:0:15038:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:0.1:1787826399.499680:0:15038:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826399.499687:0:15038:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969cc0.
00010000:00000010:0.1:1787826399.508028:0:15038:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826399.508036:0:15038:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969120.
00010000:00000010:3.1:1787826399.523710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826399.588873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fce0.
00010000:00000010:0.1:1787826399.598836:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826399.598845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826399.614484:0:15041:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84500840.
00000080:00000010:0.1:1787826399.614499:0:15041:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00000080:00000010:1.1:1787826399.647041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00010000:00000010:0.1:1787826399.651465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00010000:00000010:3.1:1787826399.657062:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:3.1:1787826399.657083:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fce0.
00010000:00000010:3.1:1787826399.672725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:1.1:1787826399.719111:0:15044:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:1.1:1787826399.752998:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707100.
00010000:00000010:1.1:1787826399.753004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826399.779988:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845039c0.
00000080:00000010:1.1:1787826399.780004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:0.1:1787826399.797019:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:0.1:1787826399.797032:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:1.1:1787826399.820703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826399.837788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:2.1:1787826399.851751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:2.1:1787826399.886932:0:15047:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599c0.
00010000:00000010:3.1:1787826399.915866:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97900.
00010000:00000010:3.1:1787826399.915874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:3.1:1787826399.933316:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845039c0.
00000080:00000010:3.1:1787826399.933329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:3.1:1787826399.986374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:3.1:1787826399.986395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:3.1:1787826400.038583:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:3.1:1787826400.038600:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459120.
00010000:00000010:0.1:1787826400.054560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:1.1:1787826400.113117:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459660.
00010000:00000010:1.1:1787826400.142717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707100.
00010000:00000010:1.1:1787826400.142726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826400.175705:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845239c0.
00000080:00000010:1.1:1787826400.175723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:1.1:1787826400.185707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826400.185726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7c0.
00010000:00000010:1.1:1787826400.197879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826400.197892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:1.1:1787826400.201824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826400.258502:0:15062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:1.1:1787826400.333979:0:15063:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:1.1:1787826400.333994:0:15063:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826400.347618:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845239c0.
00000080:00000010:1.1:1787826400.347632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:0.1:1787826400.358685:0:15063:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00010000:00000010:1.1:1787826400.358687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826400.358693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00000080:00000010:0.1:1787826400.366024:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:3.1:1787826400.386697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826400.422696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00010000:00000010:3.1:1787826400.442691:0:15066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97900.
00010000:00000010:3.1:1787826400.442700:0:15066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:3.1:1787826400.456202:0:15066:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845039c0.
00000080:00000010:3.1:1787826400.456221:0:15066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2a0.
00010000:00000010:0.1:1787826400.493714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:0.1:1787826400.493730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c80.
00010000:00000010:0.1:1787826400.501688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826400.501695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969660.
00010000:00000010:0.1:1787826400.505728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:2.1:1787826400.573791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2fc0.
00010000:00000010:1.1:1787826400.594723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826400.594737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826400.631215:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84510840.
00000080:00000010:1.1:1787826400.631234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7e0.
00010000:00000010:0.1:1787826400.664620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:1.1:1787826400.665881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969620.
00010000:00000010:1.1:1787826400.701918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826400.701939:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7e0.
00010000:00000010:2.1:1787826400.720806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:3.1:1787826400.770048:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:1.1:1787826400.782858:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826400.782873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:1.1:1787826400.792976:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514a40.
00000080:00000010:1.1:1787826400.792993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874598a0.
00010000:00000010:1.1:1787826400.826692:0:19:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826400.826701:0:19:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f0a0.
00010000:00000010:1.1:1787826400.826707:0:19:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826400.826709:0:19:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7c0.
00010000:00000010:0.1:1787826400.854618:0:15073:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826400.878221:0:15074:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:2.1:1787826400.898696:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97900.
00010000:00000010:2.1:1787826400.898703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826400.902721:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:0.1:1787826400.902735:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:1.1:1787826400.921826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826400.921835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:1.1:1787826400.924120:0:15075:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826400.930044:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:1.1:1787826400.930066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:0.1:1787826400.965702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:1.1:1787826400.976813:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826400.976841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:2.1:1787826401.015286:0:15078:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845239c0.
00000080:00000010:2.1:1787826401.015302:0:15078:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:1.1:1787826401.042799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:1.1:1787826401.042813:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:1.1:1787826401.050190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826401.050206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f0a0.
00010000:00000010:1.1:1787826401.069797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826401.101045:0:15058:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690a0.
00010000:00000010:3.1:1787826401.150764:0:15056:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97a00.
00010000:00000010:3.1:1787826401.150779:0:15056:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:0.1:1787826401.152747:0:14:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874593a0.
00010000:00000010:1.1:1787826401.176476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826401.176488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f4e0.
00010000:00000010:1.1:1787826401.185694:0:15084:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826401.194721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:3.1:1787826401.197683:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:2.1:1787826401.261927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21c0.
00010000:00000010:3.1:1787826401.287697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97a00.
00010000:00000010:3.1:1787826401.287709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:0.1:1787826401.292859:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845039c0.
00000080:00000010:0.1:1787826401.292874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c00.
00010000:00000010:1.1:1787826401.317970:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826401.317976:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:1.1:1787826401.323042:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826401.323058:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:1.1:1787826401.349257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:0.1:1787826401.404710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:1.1:1787826401.422951:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826401.422959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:1.1:1787826401.445716:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:1.1:1787826401.445730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c20.
00010000:00000010:0.1:1787826401.465924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826401.465931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ee0.
00010000:00000010:0.1:1787826401.472009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826401.477140:0:15096:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:1.1:1787826401.506189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:3.1:1787826401.538527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459040.
00010000:00000010:3.1:1787826401.556419:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:3.1:1787826401.556428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826401.563717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826401.563733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ee0.
00010000:00000010:1.1:1787826401.589925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826401.589940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826401.594684:0:15099:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826401.598880:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:1.1:1787826401.607966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:3.1:1787826401.635707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2140.
00010000:00000010:0.1:1787826401.645618:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826401.645631:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826401.647842:0:15102:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:2.1:1787826401.647852:0:15102:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:2.1:1787826401.675986:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826401.675997:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:2.1:1787826401.688519:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826401.695807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2320.
00010000:00000010:0.1:1787826401.706098:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826401.808992:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ea0.
00010000:00000010:3.1:1787826401.809852:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97a00.
00010000:00000010:3.1:1787826401.809860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:3.1:1787826401.825707:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:3.1:1787826401.825721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:1.1:1787826401.865797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826401.865810:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fce0.
00010000:00000010:1.1:1787826401.871722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826401.871737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f4e0.
00010000:00000010:2.1:1787826401.885724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826401.935781:0:15107:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:1.1:1787826401.960720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826401.960728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826401.973068:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:1.1:1787826401.973081:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:0.1:1787826402.018048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:0.1:1787826402.018062:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2040.
00010000:00000010:1.1:1787826402.059780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826402.059793:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:0.1:1787826402.084381:0:15109:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826402.118910:0:15110:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969cc0.
00010000:00000010:1.1:1787826402.144383:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826402.144402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:2.1:1787826402.186749:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84512940.
00000080:00000010:2.1:1787826402.186764:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:2.1:1787826402.207548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826402.207560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2880.
00010000:00000010:2.1:1787826402.214629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:2.1:1787826402.214677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2520.
00010000:00000010:2.1:1787826402.229995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:1.1:1787826402.262927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ffe0.
00010000:00000010:0.1:1787826402.287756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826402.287767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826402.295701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969120.
00010000:00000010:3.1:1787826402.314886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:3.1:1787826402.314898:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:3.1:1787826402.324071:0:15115:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826402.324080:0:15115:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874593a0.
00010000:00000010:1.1:1787826402.333779:0:15114:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:2.1:1787826402.391872:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:1.1:1787826402.414887:0:15126:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826402.414899:0:15126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826402.419876:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:1.1:1787826402.419895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826402.439696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826402.439713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826402.454232:0:15126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826402.454249:0:15126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f280.
00010000:00000010:0.1:1787826402.465101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:0.1:1787826402.515089:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459260.
00010000:00000010:0.1:1787826402.528883:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826402.528894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826402.534027:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:0.1:1787826402.534040:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134ee0.
00010000:00000010:1.1:1787826402.549719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826402.552825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826402.562816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826402.562824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826402.577709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:2.1:1787826402.619703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c80.
00010000:00000010:0.1:1787826402.643770:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826402.643780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:1.1:1787826402.653836:0:15132:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:1.1:1787826402.653849:0:15132:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2340.
00010000:00000010:1.1:1787826402.682010:0:15132:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826402.682018:0:15132:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826402.689778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826402.689785:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:0.1:1787826402.708878:0:15133:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826402.735740:0:15134:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134380.
00010000:00000010:3.1:1787826402.770744:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97a00.
00010000:00000010:3.1:1787826402.770763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:1.1:1787826402.783840:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826402.783855:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134380.
00010000:00000010:3.1:1787826402.819820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826402.819834:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:3.1:1787826402.828913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826402.831929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:3.1:1787826402.838900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:1.1:1787826402.888839:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:0.1:1787826402.917204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826402.917211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826402.921843:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84506300.
00000080:00000010:0.1:1787826402.921852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134ee0.
00010000:00000010:2.1:1787826402.944703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:2.1:1787826402.944714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ee0.
00010000:00000010:2.1:1787826402.952710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826402.952725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2820.
00010000:00000010:3.1:1787826402.966936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:0.1:1787826403.005190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134ee0.
00010000:00000010:3.1:1787826403.034427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43d00.
00010000:00000010:3.1:1787826403.034435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:3.1:1787826403.047904:0:15141:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84511080.
00000080:00000010:3.1:1787826403.047923:0:15141:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2960.
00010000:00000010:1.1:1787826403.092226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:1.1:1787826403.092234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2300.
00010000:00000010:1.1:1787826403.096823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826403.096832:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f0a0.
00010000:00000010:1.1:1787826403.104415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826403.126011:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2f80.
00010000:00000010:1.1:1787826403.157874:0:15144:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826403.157888:0:15144:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826403.172652:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:1.1:1787826403.172679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:1.1:1787826403.195814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826403.195827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2e0.
00010000:00000010:1.1:1787826403.202542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826403.202555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:1.1:1787826403.205959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826403.256931:0:15146:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2940.
00010000:00000010:1.1:1787826403.313285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826403.313297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:1.1:1787826403.332728:0:15120:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84504200.
00000080:00000010:1.1:1787826403.332752:0:15120:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969100.
00010000:00000010:2.1:1787826403.344747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826403.344759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2dc0.
00010000:00000010:2.1:1787826403.363939:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:2.1:1787826403.363950:0:14493:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2de0.
00010000:00000010:1.1:1787826403.385699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826403.431891:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:0.1:1787826403.452683:0:15154:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826403.452690:0:15154:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:0.1:1787826403.469712:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451dac0.
00000080:00000010:0.1:1787826403.469721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459800.
00010000:00000010:0.1:1787826403.502883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:0.1:1787826403.502892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:3.1:1787826403.508954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826403.509689:0:15159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:2.1:1787826403.528176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:1.1:1787826403.577225:0:15161:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ffe0.
00010000:00000010:1.1:1787826403.597841:0:15161:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826403.597849:0:15161:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826403.606035:0:15162:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84511080.
00000080:00000010:1.1:1787826403.606048:0:15162:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ffe0.
00010000:00000010:0.1:1787826403.624898:0:15162:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826403.624905:0:15162:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:0.1:1787826403.632716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826403.632725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:0.1:1787826403.638832:0:15162:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826403.664698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:0.1:1787826403.685532:0:15164:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826403.685544:0:15164:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826403.695797:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:0.1:1787826403.695824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:1.1:1787826403.727680:0:15158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:1.1:1787826403.727694:0:15158:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00010000:00000010:3.1:1787826403.740688:0:15165:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:3.1:1787826403.740695:0:15165:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:3.1:1787826403.748709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:1.1:1787826403.787685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:2.1:1787826403.801682:0:15167:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43c00.
00010000:00000010:2.1:1787826403.801689:0:15167:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:2.1:1787826403.812721:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84503180.
00000080:00000010:2.1:1787826403.812736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:1.1:1787826403.847856:0:15168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826403.847863:0:15168:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826403.851746:0:15168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826403.851753:0:15168:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f280.
00010000:00000010:2.1:1787826403.867102:0:15168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:0.1:1787826403.933787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459240.
00010000:00000010:2.1:1787826403.937766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43c00.
00010000:00000010:2.1:1787826403.937781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:2.1:1787826403.963422:0:15171:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451a940.
00000080:00000010:2.1:1787826403.963445:0:15171:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969aa0.
00010000:00000010:0.1:1787826403.979022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:0.1:1787826403.982680:0:15171:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2140.
00010000:00000010:0.1:1787826403.987272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826403.987286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:1.1:1787826404.006229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826404.056888:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2fc0.
00010000:00000010:2.1:1787826404.074392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97f00.
00010000:00000010:2.1:1787826404.074402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:3.1:1787826404.083124:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:3.1:1787826404.083141:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459bc0.
00010000:00000010:0.1:1787826404.133718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826404.133733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969cc0.
00010000:00000010:0.1:1787826404.141814:0:15174:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826404.141821:0:15174:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:2.1:1787826404.164701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:1.1:1787826404.195723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826404.216859:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826404.216879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:1.1:1787826404.228702:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826404.228717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:2.1:1787826404.261255:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826404.266703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459480.
00010000:00000010:0.1:1787826404.283299:0:15177:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:0.1:1787826404.283306:0:15177:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459bc0.
00010000:00000010:0.1:1787826404.291993:0:15178:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:2.1:1787826404.335856:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c80.
00010000:00000010:0.1:1787826404.389765:0:14640:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdac00.
00010000:00000010:0.1:1787826404.389781:0:14640:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826404.393145:0:14640:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:1.1:1787826404.434075:0:15185:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:1.1:1787826404.434092:0:15185:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459180.
00010000:00000010:1.1:1787826404.445468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826404.445485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:0.1:1787826404.454841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826404.512174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:3.1:1787826404.533710:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97200.
00010000:00000010:3.1:1787826404.533731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:3.1:1787826404.566717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:3.1:1787826404.566736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:1.1:1787826404.601715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:1.1:1787826404.601726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:0.1:1787826404.605780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826404.605801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:0.1:1787826404.643864:0:15193:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:2.1:1787826404.673839:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a80.
00010000:00000010:1.1:1787826404.698278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826404.698293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:1.1:1787826404.705697:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:1.1:1787826404.705714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:0.1:1787826404.735780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826404.735792:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969500.
00010000:00000010:0.1:1787826404.743585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826404.743599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:1.1:1787826404.743790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:2.1:1787826404.778063:0:15197:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ca0.
00010000:00000010:2.1:1787826404.799710:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43c00.
00010000:00000010:2.1:1787826404.799725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826404.823833:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:2.1:1787826404.823859:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ca0.
00010000:00000010:2.1:1787826404.851910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826404.851920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:0.1:1787826404.856271:0:15198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826404.856287:0:15198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:0.1:1787826404.873044:0:15198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826404.922697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:2.1:1787826404.944772:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda300.
00010000:00000010:2.1:1787826404.944787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:2.1:1787826404.966065:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520000.
00000080:00000010:2.1:1787826404.966084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874593a0.
00010000:00000010:0.1:1787826404.988084:0:15201:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:0.1:1787826404.988095:0:15201:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c60.
00010000:00000010:0.1:1787826405.003129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826405.003136:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826405.017722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826405.060375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:1.1:1787826405.071835:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826405.071849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826405.087711:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514a40.
00000080:00000010:1.1:1787826405.087727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:0.1:1787826405.119583:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826405.119591:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:1.1:1787826405.123885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:1.1:1787826405.123912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ae0.
00010000:00000010:1.1:1787826405.126830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:0.1:1787826405.158690:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:1.1:1787826405.182500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826405.182515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826405.198030:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826405.198045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:0.1:1787826405.228871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826405.228878:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c80.
00010000:00000010:0.1:1787826405.244531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826405.244538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826405.249325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:1.1:1787826405.302716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:0.1:1787826405.319775:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda300.
00010000:00000010:0.1:1787826405.319789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:0.1:1787826405.337741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451a940.
00000080:00000010:0.1:1787826405.337770:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459b80.
00010000:00000010:0.1:1787826405.353042:0:15210:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826405.353049:0:15210:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969960.
00010000:00000010:0.1:1787826405.362556:0:15210:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826405.362569:0:15210:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:0.1:1787826405.367721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:1.1:1787826405.434022:0:15183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874594c0.
00010000:00000010:0.1:1787826405.475781:0:15215:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda300.
00010000:00000010:0.1:1787826405.475794:0:15215:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826405.490716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826405.523873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826405.523885:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:0.1:1787826405.523890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826405.523893:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:1.1:1787826405.559733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826405.583719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:1.1:1787826405.609711:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826405.609726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826405.629778:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826405.629794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:0.1:1787826405.644067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:0.1:1787826405.644077:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459580.
00010000:00000010:0.1:1787826405.657727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826405.657742:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969620.
00010000:00000010:0.1:1787826405.671061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:1.1:1787826405.713883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:2.1:1787826405.726993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43c00.
00010000:00000010:2.1:1787826405.727004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:2.1:1787826405.747724:0:15228:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84516300.
00000080:00000010:2.1:1787826405.747740:0:15228:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599a0.
00010000:00000010:2.1:1787826405.781705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:2.1:1787826405.781719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2040.
00010000:00000010:2.1:1787826405.815385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826405.815404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26e0.
00010000:00000010:2.1:1787826405.820358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:1.1:1787826405.858960:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874591a0.
00010000:00000010:2.1:1787826405.875708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43c00.
00010000:00000010:2.1:1787826405.875724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:2.1:1787826405.892420:0:15231:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451d280.
00000080:00000010:2.1:1787826405.892446:0:15231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459980.
00010000:00000010:2.1:1787826405.925333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826405.925341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2dc0.
00010000:00000010:2.1:1787826405.928840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826405.928854:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:3.1:1787826405.937768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:0.1:1787826405.975931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969100.
00010000:00000010:2.1:1787826406.004933:0:2341:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97200.
00010000:00000010:2.1:1787826406.004948:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:0.1:1787826406.028714:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826406.028723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459260.
00010000:00000010:0.1:1787826406.040746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:0.1:1787826406.040763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2640.
00010000:00000010:0.1:1787826406.048923:0:15234:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826406.048929:0:15234:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826406.065720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826406.103935:0:15236:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695a0.
00010000:00000010:1.1:1787826406.124721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826406.124750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826406.152283:0:15237:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826406.152296:0:15237:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826406.183784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826406.183791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:3.1:1787826406.188710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:3.1:1787826406.188726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f280.
00010000:00000010:3.1:1787826406.224564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:0.1:1787826406.265566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:0.1:1787826406.289820:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826406.289836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:0.1:1787826406.322492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451d280.
00000080:00000010:0.1:1787826406.322513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:0.1:1787826406.340816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826406.346831:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969620.
00010000:00000010:0.1:1787826406.351703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826406.351720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695a0.
00010000:00000010:0.1:1787826406.381589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:3.1:1787826406.401487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874592e0.
00010000:00000010:0.1:1787826406.432674:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826406.432682:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826406.446870:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969120.
00010000:00000010:1.1:1787826406.465474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826406.465483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:1.1:1787826406.474318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826406.474332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:2.1:1787826406.522077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:3.1:1787826406.555415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:0.1:1787826406.575700:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826406.575717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:0.1:1787826406.606887:0:15255:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:0.1:1787826406.606901:0:15255:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:1.1:1787826406.643699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826406.643713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:2.1:1787826406.643763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:2.1:1787826406.643768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:2.1:1787826406.654911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826406.693707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2340.
00010000:00000010:1.1:1787826406.712026:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826406.712033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826406.726780:0:15258:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451d280.
00000080:00000010:1.1:1787826406.726794:0:15258:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7e0.
00010000:00000010:1.1:1787826406.762453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826406.762463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:1.1:1787826406.767684:0:15258:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826406.767691:0:15258:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7e0.
00010000:00000010:1.1:1787826406.771828:0:15259:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:0.1:1787826406.810812:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:1.1:1787826406.848800:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826406.848812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826406.853741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84524200.
00000080:00000010:1.1:1787826406.853758:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:0.1:1787826406.880697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:1.1:1787826406.885180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459120.
00010000:00000010:1.1:1787826406.889700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826406.889710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:2.1:1787826406.910703:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:0.1:1787826406.963039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:0.1:1787826406.970845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826406.970857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826406.990201:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826406.990214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:1.1:1787826407.021011:0:15265:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:1.1:1787826407.021023:0:15265:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459660.
00010000:00000010:1.1:1787826407.027698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826407.027706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:0.1:1787826407.045899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826407.129525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c80.
00010000:00000010:0.1:1787826407.161697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826407.161800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826407.167701:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826407.167715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a80.
00010000:00000010:3.1:1787826407.204808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:3.1:1787826407.204825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459240.
00010000:00000010:3.1:1787826407.213695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:3.1:1787826407.213705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d20.
00010000:00000010:2.1:1787826407.229776:0:14640:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:2.1:1787826407.266027:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:0.1:1787826407.291967:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826407.291978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826407.311710:0:15271:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826407.311724:0:15271:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:1.1:1787826407.356916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826407.356922:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:1.1:1787826407.366701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826407.366712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:2.1:1787826407.370722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826407.464738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:3.1:1787826407.484796:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97d00.
00010000:00000010:3.1:1787826407.484807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:1.1:1787826407.495073:0:15274:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:1.1:1787826407.525345:0:15276:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826407.525352:0:15276:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:0.1:1787826407.562117:0:12446:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826407.562129:0:12446:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:0.1:1787826407.569724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826407.642708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826407.658704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826407.658720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826407.668105:0:15286:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84507380.
00000080:00000010:1.1:1787826407.668120:0:15286:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:0.1:1787826407.705736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826407.705751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00010000:00000010:0.1:1787826407.714704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826407.714720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:0.1:1787826407.734720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826407.802185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:1.1:1787826407.823952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826407.823968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:1.1:1787826407.851697:0:15289:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:1.1:1787826407.851708:0:15289:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:1.1:1787826407.861758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826407.871822:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:1.1:1787826407.888690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826407.888697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:2.1:1787826407.896670:0:14640:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826407.917807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:2.1:1787826407.931789:0:15291:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826407.931796:0:15291:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:2.1:1787826407.941136:0:15292:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:2.1:1787826407.941303:0:15292:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599c0.
00010000:00000010:2.1:1787826407.964055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826407.967807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2140.
00010000:00000010:2.1:1787826407.983189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:2.1:1787826407.983195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:1.1:1787826408.009265:0:15293:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:1.1:1787826408.050712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:0.1:1787826408.059794:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826408.059807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:0.1:1787826408.064859:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520840.
00000080:00000010:0.1:1787826408.064873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2820.
00010000:00000010:2.1:1787826408.090868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:0.1:1787826408.090986:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e80.
00010000:00000010:0.1:1787826408.100807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826408.100819:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695a0.
00010000:00000010:0.1:1787826408.115882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826408.162737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:1.1:1787826408.179714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826408.179728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826408.191252:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826408.191268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:3.1:1787826408.222705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:3.1:1787826408.222719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459800.
00010000:00000010:3.1:1787826408.237706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:3.1:1787826408.237717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599c0.
00010000:00000010:0.1:1787826408.253710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:1.1:1787826408.289694:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fc40.
00010000:00000010:2.1:1787826408.304910:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826408.304920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826408.315711:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:2.1:1787826408.315725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2640.
00010000:00000010:0.1:1787826408.338110:0:15301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826408.338118:0:15301:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:0.1:1787826408.343968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826408.346985:0:15301:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:1.1:1787826408.363114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:3.1:1787826408.390966:0:15303:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:3.1:1787826408.437797:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97d00.
00010000:00000010:3.1:1787826408.437810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:3.1:1787826408.465217:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520840.
00000080:00000010:3.1:1787826408.465234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ee0.
00010000:00000010:1.1:1787826408.493541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826408.501861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:1.1:1787826408.507536:0:15304:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826408.507544:0:15304:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:1.1:1787826408.520224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826408.562457:0:15306:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969720.
00010000:00000010:0.1:1787826408.612298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826408.612309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:1.1:1787826408.632641:0:15278:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826408.633035:0:15278:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969720.
00010000:00000010:0.1:1787826408.679096:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:0.1:1787826408.679122:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459b60.
00010000:00000010:2.1:1787826408.681717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:2.1:1787826408.681729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:0.1:1787826408.719244:0:15307:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826408.789700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fc40.
00010000:00000010:1.1:1787826408.790847:0:15317:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826408.790866:0:15317:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826408.818724:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84516300.
00000080:00000010:1.1:1787826408.818738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:0.1:1787826408.849695:0:15319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826408.851162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:0.1:1787826408.854790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:0.1:1787826408.854803:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2f80.
00010000:00000010:1.1:1787826408.896940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826408.981043:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:3.1:1787826409.036001:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97d00.
00010000:00000010:3.1:1787826409.036015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:3.1:1787826409.042785:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:3.1:1787826409.042801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:0.1:1787826409.058984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826409.077034:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:0.1:1787826409.084198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826409.084213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:0.1:1787826409.084700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:1.1:1787826409.158711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:3.1:1787826409.179712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97d00.
00010000:00000010:3.1:1787826409.179724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:3.1:1787826409.209700:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514a40.
00000080:00000010:3.1:1787826409.209712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:0.1:1787826409.226717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826409.226728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2340.
00010000:00000010:0.1:1787826409.231719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826409.231732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:1.1:1787826409.249051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:2.1:1787826409.294032:0:15327:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:2.1:1787826409.314844:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826409.314850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826409.323704:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520000.
00000080:00000010:2.1:1787826409.323719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21c0.
00010000:00000010:1.1:1787826409.337898:0:15328:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826409.337904:0:15328:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826409.342716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826409.342732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:1.1:1787826409.365979:0:15329:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:0.1:1787826409.402010:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:0.1:1787826409.429701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826409.429708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:0.1:1787826409.429715:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826409.429723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:3.1:1787826409.445930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:3.1:1787826409.445938:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459480.
00010000:00000010:3.1:1787826409.459745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:3.1:1787826409.459753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:1.1:1787826409.470243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:2.1:1787826409.501807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:2.1:1787826409.526001:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826409.526009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:2.1:1787826409.531960:0:15334:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:2.1:1787826409.531973:0:15334:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459180.
00000080:00000010:1.1:1787826409.555778:0:15334:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459480.
00010000:00000010:0.1:1787826409.564282:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00010000:00000010:1.1:1787826409.568711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826409.568725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826409.583861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826409.620013:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2140.
00010000:00000010:3.1:1787826409.635718:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97d00.
00010000:00000010:3.1:1787826409.635732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:3.1:1787826409.651136:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:3.1:1787826409.652009:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459fc0.
00010000:00000010:0.1:1787826409.682882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:1.1:1787826409.683049:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00010000:00000010:1.1:1787826409.690726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826409.690737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826409.703718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:2.1:1787826409.746818:0:15339:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:2.1:1787826409.776080:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826409.776089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:0.1:1787826409.806112:0:15312:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826409.806127:0:15312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2040.
00010000:00000010:2.1:1787826409.834043:0:15340:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:2.1:1787826409.834056:0:15340:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459b80.
00010000:00000010:2.1:1787826409.834062:0:15340:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826409.834065:0:15340:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e23c0.
00010000:00000010:2.1:1787826409.853732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:0.1:1787826409.906954:0:15347:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:2.1:1787826409.939532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826409.939545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:2.1:1787826409.946713:0:15352:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:2.1:1787826409.946727:0:15352:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:2.1:1787826409.968747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826409.968760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:2.1:1787826409.979033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826409.979045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c80.
00010000:00000010:0.1:1787826410.016703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:1.1:1787826410.040888:0:15354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:0.1:1787826410.075755:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826410.075768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826410.088793:0:15355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:0.1:1787826410.088807:0:15355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:1.1:1787826410.111832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826410.111847:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:1.1:1787826410.132862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826410.132875:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826410.143197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:0.1:1787826410.172748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00010000:00000010:1.1:1787826410.199977:0:15357:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826410.199988:0:15357:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826410.226884:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:1.1:1787826410.226901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:0.1:1787826410.245458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826410.245481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00000080:00000010:2.1:1787826410.254019:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00010000:00000010:0.1:1787826410.254712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00010000:00000010:2.1:1787826410.278754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:1.1:1787826410.306640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:2.1:1787826410.350802:0:14640:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826410.350827:0:14640:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826410.365702:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:2.1:1787826410.365723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:1.1:1787826410.394849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:0.1:1787826410.401711:0:15361:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826410.406495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826410.409945:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:0.1:1787826410.430994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826410.474579:0:15363:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:0.1:1787826410.508714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826410.508727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826410.522090:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:0.1:1787826410.522107:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:1.1:1787826410.535800:0:15364:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826410.535818:0:15364:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:1.1:1787826410.546722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826410.546743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:1.1:1787826410.562976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826410.603072:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826410.632812:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826410.632825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826410.642722:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826410.642740:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ae0.
00010000:00000010:3.1:1787826410.661988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:3.1:1787826410.662010:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:0.1:1787826410.676711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826410.676729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ee0.
00010000:00000010:2.1:1787826410.725215:0:15368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826410.760021:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:1.1:1787826410.774799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826410.774819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826410.819899:0:15379:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826410.819922:0:15379:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:3.1:1787826410.831101:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:3.1:1787826410.831123:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e80.
00010000:00000010:0.1:1787826410.841994:0:15375:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:0.1:1787826410.842008:0:15375:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459800.
00010000:00000010:1.1:1787826410.851885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:3.1:1787826410.901825:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d20.
00010000:00000010:1.1:1787826410.922715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826410.922724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:1.1:1787826410.932929:0:15382:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826410.932950:0:15382:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00010000:00000010:1.1:1787826410.959717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826410.959739:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:1.1:1787826410.977962:0:15382:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826410.977971:0:15382:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:0.1:1787826410.987579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826411.032784:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:1.1:1787826411.050205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826411.050213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826411.066201:0:15379:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826411.066219:0:15379:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:0.1:1787826411.093706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826411.093716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969960.
00010000:00000010:1.1:1787826411.096783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:1.1:1787826411.096790:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826411.102007:0:15385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826411.136945:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:0.1:1787826411.156052:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826411.156076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826411.174512:0:15388:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:0.1:1787826411.184764:0:15388:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:0.1:1787826411.199714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826411.199728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00000080:00000010:1.1:1787826411.208756:0:15388:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826411.209710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00010000:00000010:1.1:1787826411.216714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826411.262378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694c0.
00010000:00000010:0.1:1787826411.275791:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826411.275800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826411.282764:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:0.1:1787826411.282776:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:1.1:1787826411.320792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826411.322264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:0.1:1787826411.337355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826411.340173:0:15391:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:0.1:1787826411.348765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:1.1:1787826411.391687:0:15393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:0.1:1787826411.404771:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826411.404784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826411.416408:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84511080.
00000080:00000010:0.1:1787826411.416423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:2.1:1787826411.446110:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826411.446116:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2dc0.
00010000:00000010:2.1:1787826411.459989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826411.460004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:2.1:1787826411.472776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:0.1:1787826411.515847:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:3.1:1787826411.536520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97300.
00010000:00000010:3.1:1787826411.536536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:2.1:1787826411.555717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84511080.
00000080:00000010:2.1:1787826411.555732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:1.1:1787826411.591715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826411.591741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826411.604723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826411.604738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:0.1:1787826411.619699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826411.684354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969500.
00010000:00000010:0.1:1787826411.736055:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826411.736068:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:0.1:1787826411.736078:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874594c0.
00010000:00000010:1.1:1787826411.768108:0:15405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:1.1:1787826411.768196:0:15405:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:1.1:1787826411.774916:0:15405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826411.774924:0:15405:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f5c0.
00010000:00000010:1.1:1787826411.788700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:0.1:1787826411.829707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826411.865177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826411.865197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:1.1:1787826411.878800:0:15412:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826411.878816:0:15412:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:2.1:1787826411.906961:0:15412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:2.1:1787826411.906969:0:15412:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:2.1:1787826411.914019:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826411.914027:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:3.1:1787826411.923995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826411.980161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826411.991703:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826411.991719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826412.027725:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826412.027736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00010000:00000010:1.1:1787826412.069867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:1.1:1787826412.069888:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:1.1:1787826412.102151:0:15415:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826412.102158:0:15415:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:2.1:1787826412.124713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826412.167782:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c80.
00010000:00000010:0.1:1787826412.205282:0:15418:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826412.205294:0:15418:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:0.1:1787826412.222695:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451a940.
00000080:00000010:0.1:1787826412.222708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00010000:00000010:1.1:1787826412.248263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826412.248277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826412.261694:0:15418:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826412.261701:0:15418:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:1.1:1787826412.267826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826412.300707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:2.1:1787826412.311895:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97d00.
00010000:00000010:2.1:1787826412.311910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:0.1:1787826412.329098:0:15421:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520000.
00000080:00000010:0.1:1787826412.329117:0:15421:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:2.1:1787826412.351851:0:15421:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826412.351930:0:15421:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00000080:00000010:0.1:1787826412.362700:0:15421:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:2.1:1787826412.366722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00010000:00000010:0.1:1787826412.370147:0:15421:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:1.1:1787826412.400810:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:3.1:1787826412.432457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97300.
00010000:00000010:3.1:1787826412.432473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:3.1:1787826412.473597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:3.1:1787826412.473615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:1.1:1787826412.498746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826412.498760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:1.1:1787826412.512696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826412.512704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:0.1:1787826412.516741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:1.1:1787826412.556718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:2.1:1787826412.571753:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43600.
00010000:00000010:2.1:1787826412.571767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:2.1:1787826412.574960:0:15427:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:2.1:1787826412.574973:0:15427:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:0.1:1787826412.597123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:0.1:1787826412.597129:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874591a0.
00010000:00000010:0.1:1787826412.605845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826412.610709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969100.
00010000:00000010:0.1:1787826412.614723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826412.641705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:0.1:1787826412.661416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826412.661439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826412.693682:0:15432:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:3.1:1787826412.721145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826412.721156:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874592e0.
00010000:00000010:1.1:1787826412.730458:0:15439:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:1.1:1787826412.730467:0:15439:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d20.
00010000:00000010:0.1:1787826412.740873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826412.791786:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:3.1:1787826412.805781:0:15441:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97300.
00010000:00000010:3.1:1787826412.807527:0:15441:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:3.1:1787826412.828920:0:15442:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84516300.
00000080:00000010:3.1:1787826412.828937:0:15442:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26c0.
00010000:00000010:0.1:1787826412.850854:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826412.850868:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:0.1:1787826412.850878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826412.850881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:1.1:1787826412.871900:0:15442:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826412.905725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:2.1:1787826412.929708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43600.
00010000:00000010:2.1:1787826412.929720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826412.934698:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:2.1:1787826412.934712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00000080:00000010:0.1:1787826412.948832:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459040.
00010000:00000010:1.1:1787826412.953695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00010000:00000010:0.1:1787826412.957683:0:15445:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826412.957783:0:15445:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:0.1:1787826412.974710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:3.1:1787826412.999218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:0.1:1787826413.026723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826413.026747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:0.1:1787826413.037892:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826413.037914:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:3.1:1787826413.098881:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:3.1:1787826413.098895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00010000:00000010:1.1:1787826413.106774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:1.1:1787826413.106796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874590e0.
00010000:00000010:0.1:1787826413.126575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826413.170745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:1.1:1787826413.194756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826413.194770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826413.220057:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826413.220071:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:3.1:1787826413.256075:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826413.256084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459b60.
00010000:00000010:3.1:1787826413.262679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:2.1:1787826413.262699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ea0.
00010000:00000010:0.1:1787826413.266834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826413.338698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:1.1:1787826413.357830:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826413.357861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:2.1:1787826413.368732:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:2.1:1787826413.368746:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2e0.
00010000:00000010:0.1:1787826413.391710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826413.391725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:0.1:1787826413.411146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826413.411162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826413.415134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826413.463753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:0.1:1787826413.484754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826413.484768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826413.493717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826413.493731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ee0.
00010000:00000010:1.1:1787826413.516767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826413.529823:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:1.1:1787826413.535799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826413.535813:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:1.1:1787826413.554144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:2.1:1787826413.629705:0:15462:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:1.1:1787826413.641081:0:14493:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826413.641092:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826413.688188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:1.1:1787826413.703813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826413.703831:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:1.1:1787826413.723863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826413.723877:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f7a0.
00010000:00000010:1.1:1787826413.744878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826413.794721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969720.
00010000:00000010:1.1:1787826413.809701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826413.809723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826413.825707:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826413.825726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:2.1:1787826413.852705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826413.852720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:0.1:1787826413.862690:0:15472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826413.862705:0:15472:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969720.
00010000:00000010:0.1:1787826413.883795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:1.1:1787826413.906562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:2.1:1787826413.927866:0:15475:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43600.
00010000:00000010:2.1:1787826413.927884:0:15475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:3.1:1787826413.939825:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:3.1:1787826413.939834:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459bc0.
00010000:00000010:1.1:1787826413.957718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:1.1:1787826413.957734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:1.1:1787826413.965811:0:15475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826413.965818:0:15475:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:1.1:1787826413.981705:0:15475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:3.1:1787826414.010816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:2.1:1787826414.030740:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97300.
00010000:00000010:2.1:1787826414.030757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:0.1:1787826414.034376:0:15479:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:0.1:1787826414.034398:0:15479:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874590e0.
00010000:00000010:1.1:1787826414.057758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826414.057777:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:1.1:1787826414.082735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:2.1:1787826414.086832:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:3.1:1787826414.114860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826414.159300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:0.1:1787826414.188725:0:15482:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826414.188736:0:15482:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826414.208392:0:15482:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826414.208417:0:15482:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:1.1:1787826414.226099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826414.230080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826414.237123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:2.1:1787826414.242039:0:1343:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:0.1:1787826414.255728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:1.1:1787826414.295987:0:15484:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:1.1:1787826414.322055:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826414.322069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826414.341705:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826414.341717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:3.1:1787826414.376154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:0.1:1787826414.379912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459180.
00010000:00000010:1.1:1787826414.389575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826414.395715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826414.407354:0:15485:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826414.469840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:2.1:1787826414.520260:0:15489:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43600.
00010000:00000010:2.1:1787826414.520278:0:15489:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:1.1:1787826414.535705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:0.1:1787826414.575078:0:15492:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00010000:00000010:1.1:1787826414.579325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826414.579344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00000080:00000010:2.1:1787826414.582748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:3.1:1787826414.583121:0:15488:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826414.642746:0:15499:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:0.1:1787826414.659841:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda500.
00010000:00000010:0.1:1787826414.659853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:0.1:1787826414.689790:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84500000.
00000080:00000010:0.1:1787826414.689800:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:2.1:1787826414.714991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:2.1:1787826414.715007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:2.1:1787826414.727483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826414.727498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:1.1:1787826414.736808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:0.1:1787826414.759946:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c80.
00010000:00000010:1.1:1787826414.783944:0:15502:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826414.783953:0:15502:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826414.789709:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84520000.
00000080:00000010:1.1:1787826414.789726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:0.1:1787826414.825979:0:15498:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826414.825994:0:15498:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:0.1:1787826414.834946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826414.834965:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:1.1:1787826414.882780:0:15504:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:1.1:1787826414.895049:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832ff40.
00010000:00000010:0.1:1787826414.911771:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826414.911790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826414.925766:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:0.1:1787826414.925784:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969960.
00010000:00000010:2.1:1787826414.951083:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:2.1:1787826414.951111:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:1.1:1787826414.960871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:1.1:1787826414.960883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:1.1:1787826414.983545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826415.027731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459020.
00010000:00000010:0.1:1787826415.057852:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826415.057873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:3.1:1787826415.084713:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:3.1:1787826415.084734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:3.1:1787826415.103396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:3.1:1787826415.103427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874593a0.
00010000:00000010:3.1:1787826415.121725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826415.125728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874591a0.
00010000:00000010:0.1:1787826415.148611:0:15509:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826415.218641:0:15498:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:0.1:1787826415.225683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826415.225712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826415.235808:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515ac0.
00000080:00000010:0.1:1787826415.235830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:1.1:1787826415.267970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826415.267979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:1.1:1787826415.279700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826415.279719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:0.1:1787826415.283161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826415.377375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:0.1:1787826415.411153:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:0.1:1787826415.411171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826415.436547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:2.1:1787826415.490069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:2.1:1787826415.490076:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2de0.
00010000:00000010:1.1:1787826415.517089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:1.1:1787826415.517108:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:2.1:1787826415.542457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826415.577701:0:15526:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:1.1:1787826415.594979:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda400.
00010000:00000010:1.1:1787826415.595012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:2.1:1787826415.628812:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:2.1:1787826415.628838:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2da0.
00010000:00000010:1.1:1787826415.671637:0:15527:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826415.671671:0:15527:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:1.1:1787826415.686711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826415.697887:0:15527:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:0.1:1787826415.707938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826415.749699:0:15529:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826415.765753:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826415.765773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826415.791767:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451dac0.
00000080:00000010:1.1:1787826415.791788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:2.1:1787826415.814730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:2.1:1787826415.814739:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:2.1:1787826415.823887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:2.1:1787826415.823896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00010000:00000010:2.1:1787826415.827982:0:15531:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:0.1:1787826415.869970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459100.
00010000:00000010:2.1:1787826415.896644:0:15533:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43600.
00010000:00000010:2.1:1787826415.896833:0:15533:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826415.903731:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451eb40.
00000080:00000010:2.1:1787826415.903748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00010000:00000010:0.1:1787826415.930721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:0.1:1787826415.933927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:0.1:1787826415.953962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826415.953976:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:2.1:1787826415.970508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826416.011380:0:14640:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459980.
00010000:00000010:1.1:1787826416.042528:0:15535:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826416.042546:0:15535:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826416.059788:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451eb40.
00000080:00000010:1.1:1787826416.059808:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:0.1:1787826416.110941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826416.115510:0:15536:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:0.1:1787826416.123689:0:15536:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826416.123699:0:15536:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969500.
00010000:00000010:0.1:1787826416.142729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:2.1:1787826416.181255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2300.
00010000:00000010:1.1:1787826416.217732:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826416.217747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826416.237814:0:15539:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826416.237833:0:15539:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:3.1:1787826416.283250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:3.1:1787826416.283272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:3.1:1787826416.289762:0:15539:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:3.1:1787826416.289777:0:15539:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459660.
00010000:00000010:2.1:1787826416.299796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:2.1:1787826416.358101:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:0.1:1787826416.367935:0:15543:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda600.
00010000:00000010:0.1:1787826416.367953:0:15543:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826416.367967:0:15543:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826416.408073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826416.408083:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:0.1:1787826416.418089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826416.418107:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:0.1:1787826416.426742:0:15543:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:2.1:1787826416.472694:0:15553:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00010000:00000010:1.1:1787826416.509146:0:15554:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826416.509167:0:15554:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:1.1:1787826416.522727:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:1.1:1787826416.522747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2520.
00010000:00000010:0.1:1787826416.543121:0:15554:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826416.543136:0:15554:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826416.554529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826416.554543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:0.1:1787826416.585970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826416.636720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:1.1:1787826416.661795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda600.
00010000:00000010:1.1:1787826416.661813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:1.1:1787826416.666719:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826416.666738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ea0.
00010000:00000010:0.1:1787826416.698886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:0.1:1787826416.698910:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d20.
00010000:00000010:1.1:1787826416.729032:0:15557:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826416.729048:0:15557:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826416.741726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:2.1:1787826416.781711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:0.1:1787826416.815533:0:15560:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:0.1:1787826416.815548:0:15560:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826416.827703:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826416.827730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929697e0.
00010000:00000010:0.1:1787826416.844009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826416.844024:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:1.1:1787826416.864109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:1.1:1787826416.871707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:0.1:1787826416.876673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826416.913708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:0.1:1787826416.957699:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:0.1:1787826416.957713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:1.1:1787826416.984502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:1.1:1787826416.984511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459480.
00010000:00000010:2.1:1787826417.013709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:2.1:1787826417.013725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:0.1:1787826417.028963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:0.1:1787826417.028981:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:2.1:1787826417.072519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826417.107743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826417.124750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826417.124760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826417.152031:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84501080.
00000080:00000010:1.1:1787826417.152038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826417.197633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:1.1:1787826417.197648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:1.1:1787826417.202711:0:15566:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826417.202717:0:15566:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826417.210791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826417.259796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826417.271775:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:0.1:1787826417.271782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826417.275831:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:1.1:1787826417.275846:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:3.1:1787826417.319549:0:15569:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826417.319560:0:15569:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459a40.
00010000:00000010:3.1:1787826417.333565:0:13135:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:3.1:1787826417.333578:0:13135:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874597e0.
00010000:00000010:3.1:1787826417.369892:0:15573:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:3.1:1787826417.408745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00010000:00000010:0.1:1787826417.435796:0:15580:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:0.1:1787826417.435808:0:15580:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826417.445680:0:15581:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84505280.
00000080:00000010:1.1:1787826417.445693:0:15581:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2dc0.
00010000:00000010:2.1:1787826417.477712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:2.1:1787826417.477728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:2.1:1787826417.487867:0:15581:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826417.487873:0:15581:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:2.1:1787826417.500912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:0.1:1787826417.579629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a1c0.
00010000:00000010:0.1:1787826417.584973:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:0.1:1787826417.584987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826417.608586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:0.1:1787826417.608613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85f15180.
00010000:00000010:1.1:1787826417.623862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826417.623876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:1.1:1787826417.632765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826417.632779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:0.1:1787826417.655094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826417.695878:0:15586:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:1.1:1787826417.722268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826417.722275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826417.742729:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:1.1:1787826417.742745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:0.1:1787826417.759710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826417.759726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296ae40.
00010000:00000010:0.1:1787826417.780582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826417.780597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296ade0.
00010000:00000010:0.1:1787826417.800511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:2.1:1787826417.850955:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f300.
00010000:00000010:1.1:1787826417.852675:0:15590:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda200.
00010000:00000010:1.1:1787826417.855463:0:15590:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:1.1:1787826417.871808:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826417.871827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a500.
00010000:00000010:1.1:1787826417.879803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826417.879822:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826417.883895:0:15590:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826417.883902:0:15590:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fe60.
00010000:00000010:1.1:1787826417.889707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:2.1:1787826417.929907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:2.1:1787826417.949698:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826417.949714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:1.1:1787826417.969730:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826417.969748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:3.1:1787826417.997975:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:3.1:1787826417.997987:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d20.
00010000:00000010:3.1:1787826418.002234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826418.002250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874594c0.
00010000:00000010:1.1:1787826418.020745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:1.1:1787826418.060225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:2.1:1787826418.080705:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826418.080733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:2.1:1787826418.094309:0:15596:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:2.1:1787826418.103917:0:15596:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d20.
00010000:00000010:0.1:1787826418.132736:0:15596:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:0.1:1787826418.132743:0:15596:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874594c0.
00010000:00000010:0.1:1787826418.136826:0:15596:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826418.145183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a8c0.
00010000:00000010:0.1:1787826418.150943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826418.177725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2940.
00010000:00000010:1.1:1787826418.207746:0:15599:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826418.207760:0:15599:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826418.215799:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826418.215819:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a500.
00010000:00000010:1.1:1787826418.245017:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826418.245024:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:1.1:1787826418.249712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:2.1:1787826418.249802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fca0.
00010000:00000010:3.1:1787826418.256835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:1.1:1787826418.285687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:3.1:1787826418.340027:0:15603:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97400.
00010000:00000010:3.1:1787826418.340040:0:15603:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:3.1:1787826418.340053:0:15603:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26c0.
00010000:00000010:0.1:1787826418.373732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826418.373742:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a1c0.
00010000:00000010:1.1:1787826418.375249:0:15604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826418.375258:0:15604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a500.
00010000:00000010:0.1:1787826418.403484:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:0.1:1787826418.441912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c20.
00010000:00000010:2.1:1787826418.457679:0:15614:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826418.457687:0:15614:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:2.1:1787826418.462484:0:15614:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:2.1:1787826418.462499:0:15614:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874599a0.
00010000:00000010:0.1:1787826418.492832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826418.492845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85eacb60.
00010000:00000010:0.1:1787826418.503141:0:15614:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826418.503148:0:15614:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296aaa0.
00010000:00000010:0.1:1787826418.507715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:1.1:1787826418.580892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:2.1:1787826418.607022:0:15617:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97400.
00010000:00000010:2.1:1787826418.607036:0:15617:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:3.1:1787826418.610813:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:3.1:1787826418.610820:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459800.
00010000:00000010:2.1:1787826418.625379:0:15617:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:2.1:1787826418.625388:0:15617:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00010000:00000010:2.1:1787826418.646712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:2.1:1787826418.646724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2940.
00010000:00000010:1.1:1787826418.653719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:1.1:1787826418.701490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a8c0.
00010000:00000010:0.1:1787826418.737539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda700.
00010000:00000010:0.1:1787826418.737555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:0.1:1787826418.742860:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826418.742889:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:1.1:1787826418.776743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826418.785519:0:15620:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826418.790806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826418.801151:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:0.1:1787826418.804789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826418.873307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861347e0.
00010000:00000010:2.1:1787826418.903831:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826418.903838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:2.1:1787826418.920713:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:2.1:1787826418.920724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:1.1:1787826418.929949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826418.933713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826418.947177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826418.947184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f580.
00010000:00000010:1.1:1787826418.957906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826419.000695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861349a0.
00010000:00000010:1.1:1787826419.022118:0:15626:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826419.022132:0:15626:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826419.026437:0:15626:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:1.1:1787826419.026451:0:15626:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:2.1:1787826419.051752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:2.1:1787826419.051767:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:0.1:1787826419.060611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826419.064403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2140.
00010000:00000010:0.1:1787826419.075706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:2.1:1787826419.110470:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fd40.
00010000:00000010:0.1:1787826419.139845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826419.139860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:0.1:1787826419.161449:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522100.
00000080:00000010:0.1:1787826419.161466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459fc0.
00010000:00000010:1.1:1787826419.188164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826419.188181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f920.
00010000:00000010:1.1:1787826419.200683:0:15629:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826419.200697:0:15629:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826419.220525:0:15630:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:0.1:1787826419.265970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134ee0.
00010000:00000010:1.1:1787826419.277701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826419.277714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:1.1:1787826419.300777:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84505ac0.
00000080:00000010:1.1:1787826419.300789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459260.
00010000:00000010:2.1:1787826419.331047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:2.1:1787826419.331063:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459980.
00010000:00000010:2.1:1787826419.344770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826419.344787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e23c0.
00010000:00000010:2.1:1787826419.358812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:3.1:1787826419.438954:0:14640:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:3.1:1787826419.465186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97000.
00010000:00000010:3.1:1787826419.465202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826419.503374:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:0.1:1787826419.542487:0:15636:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:0.1:1787826419.542499:0:15636:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ee0.
00010000:00000010:0.1:1787826419.552675:0:15644:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826419.552717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a1c0.
00010000:00000010:3.1:1787826419.574718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826419.602733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861349a0.
00010000:00000010:0.1:1787826419.617834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826419.617849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826419.628750:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:0.1:1787826419.628768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:3.1:1787826419.673897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:2.1:1787826419.680458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459de0.
00010000:00000010:1.1:1787826419.680772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:0.1:1787826419.686177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e25c0.
00010000:00000010:0.1:1787826419.713135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826419.743887:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:1.1:1787826419.760894:0:15644:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826419.760903:0:15644:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826419.760910:0:15644:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:1.1:1787826419.760917:0:15644:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:0.1:1787826419.809020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:0.1:1787826419.809036:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874591a0.
00000080:00000010:3.1:1787826419.809773:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26c0.
00010000:00000010:2.1:1787826419.809866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00010000:00000010:0.1:1787826419.832984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:2.1:1787826419.871570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:0.1:1787826419.896940:0:15652:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826419.896948:0:15652:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:0.1:1787826419.900750:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826419.900762:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459980.
00000080:00000010:3.1:1787826419.928186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e23c0.
00010000:00000010:2.1:1787826419.928943:0:2342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00010000:00000010:1.1:1787826419.938702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:1.1:1787826419.938712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874598a0.
00010000:00000010:1.1:1787826419.964709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826420.014718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:3.1:1787826420.032963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97000.
00010000:00000010:3.1:1787826420.032972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:3.1:1787826420.054580:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:3.1:1787826420.054596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2dc0.
00010000:00000010:0.1:1787826420.083998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826420.084006:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:0.1:1787826420.097572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826420.097585:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:0.1:1787826420.119474:0:15657:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:1.1:1787826420.170958:0:15658:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459260.
00010000:00000010:1.1:1787826420.179454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826420.179468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826420.179478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826420.179487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f500.
00010000:00000010:0.1:1787826420.206629:0:15659:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826420.206648:0:15659:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ae0.
00010000:00000010:0.1:1787826420.209958:0:15659:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826420.209965:0:15659:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:0.1:1787826420.219429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826420.267473:0:15661:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:0.1:1787826420.310481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97000.
00010000:00000010:0.1:1787826420.310498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826420.310505:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826420.310514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:2.1:1787826420.339707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826420.339723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:2.1:1787826420.354239:0:15662:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826420.354252:0:15662:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:2.1:1787826420.364124:0:15663:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:1.1:1787826420.393775:0:15664:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:1.1:1787826420.446740:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:1.1:1787826420.446753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826420.477689:0:15670:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f9c0.
00010000:00000010:3.1:1787826420.499962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:3.1:1787826420.499973:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459000.
00010000:00000010:3.1:1787826420.513928:0:15668:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:3.1:1787826420.513935:0:15668:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c60.
00010000:00000010:3.1:1787826420.521926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:0.1:1787826420.560718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2f80.
00010000:00000010:1.1:1787826420.576951:0:15674:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826420.576961:0:15674:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:1.1:1787826420.604715:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84506b40.
00000080:00000010:1.1:1787826420.604733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874263e0.
00010000:00000010:0.1:1787826420.629436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826420.634964:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296a1c0.
00010000:00000010:0.1:1787826420.634978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:2.1:1787826420.635706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9296aaa0.
00010000:00000010:1.1:1787826420.667254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826420.702137:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826420.714695:0:15680:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826420.714704:0:15680:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:0.1:1787826420.735887:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:0.1:1787826420.735901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd84d07c40.
00010000:00000010:0.1:1787826420.751893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826420.751908:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:0.1:1787826420.760969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:2.1:1787826420.769011:0:15680:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:2.1:1787826420.774896:0:15674:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289a00.
00000080:00000010:1.1:1787826420.836722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87426520.
00010000:00000010:0.1:1787826420.853478:0:15683:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826420.853492:0:15683:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826420.861743:0:15683:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826420.861758:0:15683:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:0.1:1787826420.897371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:0.1:1787826420.897381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ae0.
00000080:00000010:3.1:1787826420.899943:0:15683:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:0.1:1787826420.900710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00010000:00000010:3.1:1787826420.911723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:1.1:1787826420.963314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861347e0.
00010000:00000010:3.1:1787826420.996258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:3.1:1787826420.996273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:3.1:1787826421.018715:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:3.1:1787826421.018731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:1.1:1787826421.047851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826421.047861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86134380.
00010000:00000010:3.1:1787826421.057520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:2.1:1787826421.070565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861349a0.
00010000:00000010:0.1:1787826421.088804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826421.114678:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87426520.
00010000:00000010:1.1:1787826421.132983:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:1.1:1787826421.132996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:1.1:1787826421.139860:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:1.1:1787826421.139874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459740.
00010000:00000010:2.1:1787826421.175247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826421.175258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2640.
00010000:00000010:0.1:1787826421.178819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826421.178835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e21a0.
00010000:00000010:1.1:1787826421.200778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:1.1:1787826421.246023:0:15692:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd861347e0.
00010000:00000010:0.1:1787826421.272790:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826421.272802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826421.279703:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826421.279719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:2.1:1787826421.304692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826421.308071:0:15693:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:2.1:1787826421.311907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:2.1:1787826421.311919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:2.1:1787826421.324961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826421.369711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459bc0.
00010000:00000010:0.1:1787826421.384974:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826421.384995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826421.415750:0:12:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451a100.
00000080:00000010:0.1:1787826421.415782:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969720.
00000080:00000010:2.1:1787826421.445300:0:1032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969720.
00010000:00000010:1.1:1787826421.449818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00010000:00000010:0.1:1787826421.466387:0:15696:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826421.466395:0:15696:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00010000:00000010:0.1:1787826421.488606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:3.1:1787826421.545537:0:15707:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:2.1:1787826421.595754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707300.
00010000:00000010:2.1:1787826421.595769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:3.1:1787826421.603737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845018c0.
00000080:00000010:3.1:1787826421.603751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ea0.
00010000:00000010:1.1:1787826421.632812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826421.632821:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd85ee74a0.
00010000:00000010:1.1:1787826421.644733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826421.644741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd82c9e820.
00010000:00000010:0.1:1787826421.654718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826421.697888:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702a020.
00010000:00000010:2.1:1787826421.715763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826421.715771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:2.1:1787826421.736718:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845018c0.
00000080:00000010:2.1:1787826421.736734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d20.
00000080:00000010:0.1:1787826421.765975:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874594c0.
00010000:00000010:1.1:1787826421.769894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00010000:00000010:0.1:1787826421.783068:0:15711:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826421.783077:0:15711:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:1.1:1787826421.808065:0:15711:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:3.1:1787826421.842833:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702a100.
00010000:00000010:0.1:1787826421.863572:0:15713:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826421.863580:0:15713:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:1.1:1787826421.871895:0:15714:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826421.871906:0:15714:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:1.1:1787826421.890874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826421.890887:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702ab20.
00010000:00000010:1.1:1787826421.902921:0:15714:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826421.902930:0:15714:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702abe0.
00010000:00000010:0.1:1787826421.920935:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826421.943685:0:15716:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:0.1:1787826421.962485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707900.
00010000:00000010:0.1:1787826421.962503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:0.1:1787826421.978712:0:15717:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826421.978733:0:15717:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702a100.
00010000:00000010:0.1:1787826422.000709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826422.000715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:1.1:1787826422.004695:0:15717:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826422.004702:0:15717:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826422.025075:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826422.064844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:0.1:1787826422.077707:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826422.077721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826422.084743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:0.1:1787826422.084757:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969860.
00010000:00000010:1.1:1787826422.109811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:1.1:1787826422.109827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459600.
00010000:00000010:1.1:1787826422.119716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826422.119733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702ae80.
00010000:00000010:2.1:1787826422.141951:0:15721:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:0.1:1787826422.176822:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:1.1:1787826422.199219:0:15722:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:1.1:1787826422.199226:0:15722:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826422.215365:0:15723:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451b9c0.
00000080:00000010:1.1:1787826422.215381:0:15723:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702abe0.
00010000:00000010:0.1:1787826422.261858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00010000:00000010:3.1:1787826422.264636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826422.264690:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969dc0.
00000080:00000010:3.1:1787826422.264853:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ca0.
00010000:00000010:3.1:1787826422.284766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:1.1:1787826422.321301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:2.1:1787826422.353135:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:2.1:1787826422.353351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:3.1:1787826422.367482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84518840.
00000080:00000010:3.1:1787826422.367503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:1.1:1787826422.392305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826422.392325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702a020.
00010000:00000010:0.1:1787826422.398094:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:0.1:1787826422.398111:0:14493:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd8702abe0.
00010000:00000010:0.1:1787826422.421712:0:1343:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:1.1:1787826422.493688:0:15733:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:0.1:1787826422.542944:0:15738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707400.
00010000:00000010:0.1:1787826422.542963:0:15738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:0.1:1787826422.542972:0:15738:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:0.1:1787826422.542998:0:15738:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f600.
00010000:00000010:0.1:1787826422.587780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:0.1:1787826422.587798:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969080.
00010000:00000010:0.1:1787826422.599618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826422.599638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929690c0.
00010000:00000010:0.1:1787826422.630168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:3.1:1787826422.675712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:0.1:1787826422.695705:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707500.
00010000:00000010:0.1:1787826422.695721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826422.722770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826422.722787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd82c9ede0.
00000080:00000010:3.1:1787826422.754835:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2300.
00010000:00000010:2.1:1787826422.755444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00010000:00000010:0.1:1787826422.758007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:0.1:1787826422.758022:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874593a0.
00010000:00000010:0.1:1787826422.776934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:2.1:1787826422.821207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e27e0.
00010000:00000010:2.1:1787826422.826122:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826422.826133:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826422.866066:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:2.1:1787826422.866083:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2940.
00010000:00000010:2.1:1787826422.882795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826422.882827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00000080:00000010:3.1:1787826422.890862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:2.1:1787826422.891708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00010000:00000010:1.1:1787826422.906643:0:15744:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:0.1:1787826422.955990:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969500.
00010000:00000010:0.1:1787826422.962012:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdab00.
00010000:00000010:0.1:1787826422.962021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:0.1:1787826422.977702:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:0.1:1787826422.977711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:1.1:1787826422.998963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826422.999036:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2a0.
00000080:00000010:2.1:1787826423.011570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f960.
00010000:00000010:1.1:1787826423.015782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00010000:00000010:2.1:1787826423.029888:0:15748:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:1.1:1787826423.052750:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:0.1:1787826423.077846:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdab00.
00010000:00000010:0.1:1787826423.077858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826423.077865:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451e300.
00000080:00000010:0.1:1787826423.077871:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969500.
00000080:00000010:2.1:1787826423.105099:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:1.1:1787826423.110721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00010000:00000010:3.1:1787826423.114444:0:15750:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:3.1:1787826423.114455:0:15750:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:2.1:1787826423.127710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:1.1:1787826423.153883:0:15752:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:1.1:1787826423.171958:0:15753:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826423.171981:0:15753:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826423.182292:0:15753:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:1.1:1787826423.182318:0:15753:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f780.
00010000:00000010:0.1:1787826423.209763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:0.1:1787826423.209770:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:0.1:1787826423.222731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826423.222745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:0.1:1787826423.222755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826423.258714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2f80.
00010000:00000010:3.1:1787826423.286814:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:3.1:1787826423.286824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:3.1:1787826423.299739:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:3.1:1787826423.299749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874597e0.
00010000:00000010:1.1:1787826423.314821:0:15756:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826423.314827:0:15756:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2c0.
00010000:00000010:1.1:1787826423.324147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826423.324161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2e0.
00010000:00000010:1.1:1787826423.325078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826423.366704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:2.1:1787826423.381953:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826423.381962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:2.1:1787826423.400719:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:2.1:1787826423.400738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459540.
00010000:00000010:0.1:1787826423.411778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:1.1:1787826423.415140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874590e0.
00010000:00000010:1.1:1787826423.422733:0:15759:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826423.422739:0:15759:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2e0.
00010000:00000010:1.1:1787826423.432867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:3.1:1787826423.474973:0:15762:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f340.
00010000:00000010:0.1:1787826423.502599:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826423.502612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d200.
00000080:00000010:0.1:1787826423.547320:0:15769:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:0.1:1787826423.549714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826423.557375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:0.1:1787826423.557396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826423.557399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969500.
00010000:00000010:1.1:1787826423.563756:0:15772:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826423.612164:0:15773:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969760.
00010000:00000010:0.1:1787826423.638704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826423.638719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:1.1:1787826423.649872:0:15774:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826423.649887:0:15774:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:1.1:1787826423.676762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826423.676783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826423.687698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826423.687713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f080.
00010000:00000010:0.1:1787826423.715206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826423.751712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826423.764778:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826423.764792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:1.1:1787826423.768735:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826423.768748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:2.1:1787826423.789699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:2.1:1787826423.789708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c00.
00010000:00000010:0.1:1787826423.793160:0:15777:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:0.1:1787826423.793167:0:15777:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2360.
00010000:00000010:1.1:1787826423.810507:0:15777:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:2.1:1787826423.854289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459a40.
00010000:00000010:2.1:1787826423.872854:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826423.872867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288800.
00000080:00000010:2.1:1787826423.883703:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:2.1:1787826423.883716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:1.1:1787826423.908917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826423.908934:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826423.916866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826423.916879:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f6c0.
00010000:00000010:0.1:1787826423.946707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:0.1:1787826424.011861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:3.1:1787826424.031785:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:3.1:1787826424.031795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826424.039858:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451c200.
00000080:00000010:3.1:1787826424.039871:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459100.
00010000:00000010:1.1:1787826424.056758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826424.056843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832fea0.
00010000:00000010:1.1:1787826424.065861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826424.065874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd9832f2a0.
00010000:00000010:0.1:1787826424.089053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:0.1:1787826424.142707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:3.1:1787826424.166881:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:3.1:1787826424.166893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:2.1:1787826424.171360:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:2.1:1787826424.171375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459580.
00010000:00000010:0.1:1787826424.191922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826424.191936:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:0.1:1787826424.205395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826424.205411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:0.1:1787826424.216744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826424.256759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969980.
00010000:00000010:2.1:1787826424.288763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826424.288771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826424.293132:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:2.1:1787826424.293144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c00.
00010000:00000010:1.1:1787826424.303706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ca00.
00000080:00000010:0.1:1787826424.311497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459f20.
00010000:00000010:0.1:1787826424.323823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826424.323836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:0.1:1787826424.336010:0:15790:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826424.365136:0:15792:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:2.1:1787826424.410702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826424.410713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:3.1:1787826424.413727:0:15797:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:3.1:1787826424.413738:0:15797:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:3.1:1787826424.421786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826424.421795:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459740.
00010000:00000010:0.1:1787826424.436108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d400.
00000080:00000010:0.1:1787826424.436123:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c00.
00010000:00000010:0.1:1787826424.464970:0:15802:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:0.1:1787826424.493727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:1.1:1787826424.527716:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826424.527732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826424.564570:0:15804:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:1.1:1787826424.564585:0:15804:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599880.
00010000:00000010:1.1:1787826424.589401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826424.589406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599880.
00010000:00000010:0.1:1787826424.594710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826424.594728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599a20.
00010000:00000010:0.1:1787826424.605822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826424.633698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929695c0.
00010000:00000010:3.1:1787826424.646708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:3.1:1787826424.646722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:3.1:1787826424.665754:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:3.1:1787826424.665769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2340.
00010000:00000010:0.1:1787826424.679693:0:15801:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:0.1:1787826424.679719:0:15801:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:1.1:1787826424.712207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:1.1:1787826424.712218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969360.
00010000:00000010:1.1:1787826424.728698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826424.762771:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:0.1:1787826424.800721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826424.800735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:0.1:1787826424.840694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826424.840703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:1.1:1787826424.856728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:1.1:1787826424.860714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874593a0.
00010000:00000010:0.1:1787826424.872713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826424.872731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd929694a0.
00010000:00000010:0.1:1787826424.887821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:2.1:1787826424.941912:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:1.1:1787826424.953849:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826424.953865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:1.1:1787826424.973699:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826424.973714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459480.
00010000:00000010:2.1:1787826424.989423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826424.989437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:3.1:1787826424.997713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:3.1:1787826424.997729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00010000:00000010:2.1:1787826425.011710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:1.1:1787826425.059238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875998c0.
00010000:00000010:2.1:1787826425.077747:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826425.077902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:2.1:1787826425.082056:0:15816:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:2.1:1787826425.082073:0:15816:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:0.1:1787826425.117333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:0.1:1787826425.117357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ea0.
00010000:00000010:0.1:1787826425.139795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826425.139825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969a60.
00010000:00000010:2.1:1787826425.157708:0:15801:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826425.225119:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459020.
00010000:00000010:1.1:1787826425.225551:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826425.225559:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826425.265211:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826425.265240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599220.
00010000:00000010:3.1:1787826425.287704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826425.287718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c20.
00010000:00000010:3.1:1787826425.298857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:3.1:1787826425.298867:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459600.
00010000:00000010:3.1:1787826425.312854:0:15828:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:1.1:1787826425.351796:0:1032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c60.
00010000:00000010:0.1:1787826425.375921:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdab00.
00010000:00000010:0.1:1787826425.375943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:1.1:1787826425.398465:0:15831:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826425.398488:0:15831:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969700.
00010000:00000010:1.1:1787826425.423777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:3.1:1787826425.429781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599000.
00010000:00000010:0.1:1787826425.430277:0:15831:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:0.1:1787826425.430284:0:15831:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:0.1:1787826425.447063:0:15831:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826425.485820:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969900.
00010000:00000010:1.1:1787826425.506043:0:15834:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826425.506066:0:15834:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:1.1:1787826425.525382:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:1.1:1787826425.525394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459260.
00010000:00000010:2.1:1787826425.534734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:2.1:1787826425.534754:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599d60.
00010000:00000010:3.1:1787826425.544614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:2.1:1787826425.546891:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26e0.
00010000:00000010:1.1:1787826425.568229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:0.1:1787826425.626701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00010000:00000010:1.1:1787826425.655705:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826425.655726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:1.1:1787826425.668738:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:1.1:1787826425.668824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599720.
00010000:00000010:1.1:1787826425.695178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:2.1:1787826425.700227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ee0.
00010000:00000010:0.1:1787826425.715426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:0.1:1787826425.730230:0:15837:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:0.1:1787826425.734065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:1.1:1787826425.785941:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875998a0.
00010000:00000010:0.1:1787826425.794698:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826425.794720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:0.1:1787826425.802836:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84523180.
00000080:00000010:0.1:1787826425.802872:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:2.1:1787826425.815965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826425.815974:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2940.
00010000:00000010:2.1:1787826425.821514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:2.1:1787826425.824877:0:15840:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2a20.
00010000:00000010:2.1:1787826425.839941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:0.1:1787826425.859726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969ba0.
00010000:00000010:0.1:1787826425.898203:0:15843:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826425.898215:0:15843:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826425.907710:0:15843:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826425.907732:0:15843:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599720.
00010000:00000010:1.1:1787826425.928619:0:15843:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826425.928632:0:15843:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599220.
00010000:00000010:1.1:1787826425.935064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:1.1:1787826425.935084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875993c0.
00010000:00000010:1.1:1787826425.972717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:3.1:1787826426.002118:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:3.1:1787826426.012696:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:3.1:1787826426.012711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:3.1:1787826426.038967:0:15828:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874596a0.
00010000:00000010:0.1:1787826426.059548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4200.
00000080:00000010:0.1:1787826426.061111:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969c60.
00010000:00000010:0.1:1787826426.075496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4000.
00000080:00000010:0.1:1787826426.075509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969e40.
00010000:00000010:0.1:1787826426.111723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:2.1:1787826426.140083:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459e60.
00010000:00000010:2.1:1787826426.152745:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97b00.
00010000:00000010:2.1:1787826426.152753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:0.1:1787826426.172711:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826426.172728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:2.1:1787826426.192269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:2.1:1787826426.192292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e80.
00010000:00000010:2.1:1787826426.192303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826426.192306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2640.
00010000:00000010:2.1:1787826426.201223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:1.1:1787826426.257713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c80.
00010000:00000010:1.1:1787826426.293719:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826426.293732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826426.317544:0:15861:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451b180.
00000080:00000010:1.1:1787826426.317563:0:15861:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599f00.
00010000:00000010:1.1:1787826426.354001:0:15856:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826426.354022:0:15856:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599f00.
00010000:00000010:1.1:1787826426.366683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826426.366869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875998e0.
00010000:00000010:1.1:1787826426.374956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:3.1:1787826426.420899:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874598a0.
00010000:00000010:0.1:1787826426.446711:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826426.446729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826426.455019:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:0.1:1787826426.455041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399a20.
00010000:00000010:0.1:1787826426.491898:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:0.1:1787826426.491912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399c60.
00010000:00000010:1.1:1787826426.507037:0:15864:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5200.
00000080:00000010:1.1:1787826426.507050:0:15864:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863991e0.
00010000:00000010:1.1:1787826426.518804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826426.544003:0:15866:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399e80.
00010000:00000010:2.1:1787826426.570781:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826426.570794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289c00.
00000080:00000010:2.1:1787826426.594867:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:2.1:1787826426.594883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2c60.
00010000:00000010:1.1:1787826426.614517:0:15867:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:1.1:1787826426.623834:0:15867:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459540.
00010000:00000010:1.1:1787826426.640735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826426.640757:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599b00.
00010000:00000010:0.1:1787826426.661866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:1.1:1787826426.679870:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:1.1:1787826426.697763:0:15870:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707e00.
00010000:00000010:1.1:1787826426.697786:0:15870:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826426.711639:0:15870:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84504a40.
00000080:00000010:1.1:1787826426.711682:0:15870:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599020.
00010000:00000010:1.1:1787826426.750039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826426.750055:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875999c0.
00010000:00000010:1.1:1787826426.753952:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826426.753972:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599ae0.
00010000:00000010:0.1:1787826426.781271:0:15870:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:2.1:1787826426.827479:0:15872:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e22a0.
00010000:00000010:1.1:1787826426.856051:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707e00.
00010000:00000010:1.1:1787826426.856063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:0.1:1787826426.874795:0:15873:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826426.874815:0:15873:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459fc0.
00000080:00000010:3.1:1787826426.913146:0:2342:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399e80.
00010000:00000010:0.1:1787826426.925730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00010000:00000010:2.1:1787826426.951737:0:2341:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:2.1:1787826426.951886:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874591a0.
00010000:00000010:3.1:1787826426.967708:0:15876:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:1.1:1787826427.014263:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599200.
00010000:00000010:0.1:1787826427.032684:0:14640:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826427.032697:0:14640:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:3.1:1787826427.041135:0:15884:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:3.1:1787826427.041155:0:15884:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459580.
00010000:00000010:0.1:1787826427.082539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826427.082558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399e20.
00010000:00000010:0.1:1787826427.097802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826427.097813:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399d00.
00010000:00000010:0.1:1787826427.105897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826427.143774:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2de0.
00010000:00000010:0.1:1787826427.173957:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:0.1:1787826427.173983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826427.201597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:0.1:1787826427.201622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863996e0.
00010000:00000010:2.1:1787826427.229708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:2.1:1787826427.229725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26e0.
00010000:00000010:0.1:1787826427.242029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288a00.
00000080:00000010:1.1:1787826427.242926:0:15885:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:1.1:1787826427.273731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:0.1:1787826427.313824:0:15890:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399300.
00010000:00000010:1.1:1787826427.328480:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda100.
00010000:00000010:1.1:1787826427.328494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:1.1:1787826427.335148:0:15891:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826427.335174:0:15891:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863996e0.
00010000:00000010:1.1:1787826427.367859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826427.367867:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599300.
00010000:00000010:1.1:1787826427.376756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826427.376780:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599240.
00010000:00000010:0.1:1787826427.381435:0:15891:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826427.425929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399500.
00010000:00000010:1.1:1787826427.475671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707e00.
00010000:00000010:1.1:1787826427.475680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826427.480115:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:1.1:1787826427.480144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599e20.
00010000:00000010:0.1:1787826427.498740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826427.498749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863993c0.
00010000:00000010:0.1:1787826427.510760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826427.510768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399300.
00010000:00000010:0.1:1787826427.512715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826427.523781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459260.
00010000:00000010:2.1:1787826427.542345:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826427.542367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:2.1:1787826427.565879:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:2.1:1787826427.565910:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459c00.
00010000:00000010:0.1:1787826427.591320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826427.591340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863993c0.
00010000:00000010:0.1:1787826427.600775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826427.600793:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399300.
00010000:00000010:1.1:1787826427.638728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826427.669451:0:15899:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d00.
00010000:00000010:0.1:1787826427.693862:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826427.693889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826427.714495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826427.721860:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399300.
00010000:00000010:1.1:1787826427.775760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826427.775782:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599300.
00010000:00000010:1.1:1787826427.782809:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826427.782819:0:14493:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599a40.
00010000:00000010:0.1:1787826427.796921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee600.
00000080:00000010:1.1:1787826427.840981:0:15911:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599b00.
00010000:00000010:0.1:1787826427.867708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826427.867724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c000.
00000080:00000010:1.1:1787826427.885911:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826427.885923:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459100.
00010000:00000010:0.1:1787826427.913397:0:15912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826427.913421:0:15912:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399920.
00010000:00000010:0.1:1787826427.920032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826427.920041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863997c0.
00010000:00000010:0.1:1787826427.931716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826427.956706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599a00.
00010000:00000010:0.1:1787826427.968707:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826427.968723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826427.983116:0:15915:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826427.983131:0:15915:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399920.
00010000:00000010:0.1:1787826428.000704:0:15915:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:0.1:1787826428.000710:0:15915:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399ec0.
00010000:00000010:0.1:1787826428.005749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4400.
00000080:00000010:0.1:1787826428.010872:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399920.
00010000:00000010:1.1:1787826428.035932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5e00.
00000080:00000010:1.1:1787826428.079818:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399bc0.
00010000:00000010:2.1:1787826428.104298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43100.
00010000:00000010:2.1:1787826428.104307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c800.
00000080:00000010:2.1:1787826428.114787:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:2.1:1787826428.114807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459480.
00010000:00000010:1.1:1787826428.150765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:1.1:1787826428.150778:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599a00.
00010000:00000010:1.1:1787826428.161653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826428.161679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599020.
00010000:00000010:1.1:1787826428.171727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:3.1:1787826428.224709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459740.
00010000:00000010:0.1:1787826428.246947:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826428.246962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:1.1:1787826428.276098:0:15921:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:1.1:1787826428.276119:0:15921:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399be0.
00010000:00000010:0.1:1787826428.294948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:0.1:1787826428.294964:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399360.
00010000:00000010:1.1:1787826428.307588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4e00.
00000080:00000010:1.1:1787826428.307611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863998c0.
00010000:00000010:1.1:1787826428.317724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:2.1:1787826428.358394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:3.1:1787826428.380728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707000.
00010000:00000010:3.1:1787826428.380760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:0.1:1787826428.401056:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84522940.
00000080:00000010:0.1:1787826428.401079:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875991a0.
00010000:00000010:2.1:1787826428.427099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6ce00.
00000080:00000010:3.1:1787826428.427894:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459020.
00010000:00000010:0.1:1787826428.431045:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c200.
00000080:00000010:0.1:1787826428.431067:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459ee0.
00010000:00000010:0.1:1787826428.447406:0:15925:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4c00.
00000080:00000010:2.1:1787826428.488108:0:15927:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459580.
00010000:00000010:3.1:1787826428.523693:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43600.
00010000:00000010:3.1:1787826428.523701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288600.
00000080:00000010:0.1:1787826428.530827:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:0.1:1787826428.530848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:1.1:1787826428.546883:0:15928:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:1.1:1787826428.546892:0:15928:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599360.
00010000:00000010:1.1:1787826428.557835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:1.1:1787826428.557851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599e60.
00010000:00000010:0.1:1787826428.575690:0:15906:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefc00.
00000080:00000010:0.1:1787826428.621571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874592e0.
00010000:00000010:2.1:1787826428.691981:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826428.691999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feea00.
00000080:00000010:2.1:1787826428.692012:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84517380.
00000080:00000010:2.1:1787826428.692019:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875993a0.
00010000:00000010:1.1:1787826428.711701:0:15940:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6de00.
00000080:00000010:1.1:1787826428.711709:0:15940:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd874591a0.
00010000:00000010:3.1:1787826428.723745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee000.
00000080:00000010:3.1:1787826428.723769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599ec0.
00010000:00000010:2.1:1787826428.744856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d600.
00000080:00000010:3.1:1787826428.785692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2de0.
00010000:00000010:1.1:1787826428.809487:0:15943:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707f00.
00010000:00000010:1.1:1787826428.809508:0:15943:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826428.821702:0:15943:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84502100.
00000080:00000010:1.1:1787826428.821723:0:15943:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599620.
00010000:00000010:1.1:1787826428.847721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef200.
00000080:00000010:1.1:1787826428.847746:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599620.
00010000:00000010:3.1:1787826428.861825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288c00.
00000080:00000010:3.1:1787826428.861848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2f80.
00010000:00000010:2.1:1787826428.883651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6da00.
00000080:00000010:0.1:1787826428.931697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399e40.
00010000:00000010:2.1:1787826428.957872:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:2.1:1787826428.957893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6dc00.
00000080:00000010:2.1:1787826428.980864:0:15946:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84514200.
00000080:00000010:2.1:1787826428.980930:0:15946:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459d20.
00010000:00000010:2.1:1787826429.030847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:2.1:1787826429.030867:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2640.
00010000:00000010:2.1:1787826429.053430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:2.1:1787826429.053439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ca0.
00010000:00000010:2.1:1787826429.078707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:1.1:1787826429.107082:0:15948:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599cc0.
00010000:00000010:0.1:1787826429.138241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707e00.
00010000:00000010:0.1:1787826429.138272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef000.
00000080:00000010:0.1:1787826429.156350:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd8451f380.
00000080:00000010:0.1:1787826429.156372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599cc0.
00010000:00000010:2.1:1787826429.178742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:2.1:1787826429.178759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2240.
00010000:00000010:2.1:1787826429.186769:0:15949:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826429.186779:0:15949:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e20e0.
00010000:00000010:2.1:1787826429.193714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:3.1:1787826429.252784:0:15951:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e26c0.
00010000:00000010:1.1:1787826429.286692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707000.
00010000:00000010:1.1:1787826429.286700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d800.
00000080:00000010:1.1:1787826429.297877:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84518000.
00000080:00000010:1.1:1787826429.297894:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459540.
00010000:00000010:0.1:1787826429.317692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826429.325257:0:15952:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399d60.
00010000:00000010:0.1:1787826429.332897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:0.1:1787826429.332907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399aa0.
00010000:00000010:0.1:1787826429.347708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:3.1:1787826429.403710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2e00.
00010000:00000010:1.1:1787826429.449342:0:15955:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707000.
00010000:00000010:1.1:1787826429.449353:0:15955:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:1.1:1787826429.461906:0:15955:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84502100.
00000080:00000010:1.1:1787826429.461918:0:15955:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ce0.
00010000:00000010:1.1:1787826429.521795:0:261:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feee00.
00000080:00000010:1.1:1787826429.521801:0:261:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599d80.
00010000:00000010:0.1:1787826429.528468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef800.
00000080:00000010:0.1:1787826429.528483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599360.
00010000:00000010:0.1:1787826429.532713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:0.1:1787826429.638556:0:15963:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599980.
00010000:00000010:3.1:1787826429.648943:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97100.
00010000:00000010:3.1:1787826429.648957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5400.
00000080:00000010:3.1:1787826429.678842:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845118c0.
00000080:00000010:3.1:1787826429.678851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399740.
00010000:00000010:3.1:1787826429.704235:0:15967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5c00.
00000080:00000010:3.1:1787826429.704248:0:15967:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399d60.
00000080:00000010:2.1:1787826429.723237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459f20.
00010000:00000010:3.1:1787826429.728840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00010000:00000010:0.1:1787826429.732723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289600.
00000080:00000010:2.1:1787826429.772894:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599e40.
00010000:00000010:1.1:1787826429.799960:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707000.
00010000:00000010:1.1:1787826429.799978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:0.1:1787826429.817726:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:0.1:1787826429.817738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd875995c0.
00010000:00000010:3.1:1787826429.839353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289400.
00000080:00000010:3.1:1787826429.839371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2100.
00010000:00000010:1.1:1787826429.866861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6d000.
00000080:00000010:1.1:1787826429.866869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459bc0.
00010000:00000010:1.1:1787826429.886858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:2.1:1787826429.954958:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2640.
00010000:00000010:1.1:1787826429.962942:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826429.962951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6cc00.
00000080:00000010:1.1:1787826429.968027:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:1.1:1787826429.968046:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459740.
00010000:00000010:1.1:1787826430.022099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4a00.
00000080:00000010:1.1:1787826430.022117:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399080.
00010000:00000010:3.1:1787826430.042762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fef400.
00000080:00000010:2.1:1787826430.043652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599660.
00010000:00000010:0.1:1787826430.076726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289e00.
00000080:00000010:0.1:1787826430.154763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863997e0.
00010000:00000010:3.1:1787826430.182089:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97100.
00010000:00000010:3.1:1787826430.182112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288e00.
00000080:00000010:3.1:1787826430.196866:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:3.1:1787826430.196886:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2460.
00010000:00000010:0.1:1787826430.233708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:0.1:1787826430.233719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2bc0.
00010000:00000010:0.1:1787826430.244824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826430.244845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863997e0.
00010000:00000010:1.1:1787826430.247981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5000.
00000080:00000010:0.1:1787826430.312819:0:15978:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863995e0.
00010000:00000010:2.1:1787826430.368306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97100.
00010000:00000010:2.1:1787826430.368328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:3.1:1787826430.383845:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84525280.
00000080:00000010:3.1:1787826430.383864:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2dc0.
00010000:00000010:0.1:1787826430.405251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826430.405272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399b40.
00010000:00000010:0.1:1787826430.409864:0:15979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826430.409872:0:15979:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399420.
00010000:00000010:0.1:1787826430.421793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:2.1:1787826430.483774:0:15982:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2d60.
00010000:00000010:0.1:1787826430.538904:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bdae00.
00010000:00000010:0.1:1787826430.538928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289200.
00000080:00000010:2.1:1787826430.547738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2300.
00010000:00000010:0.1:1787826430.601715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826430.601723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399f60.
00000080:00000010:1.1:1787826430.619856:0:15991:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399b40.
00010000:00000010:0.1:1787826430.620693:0:15985:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00010000:00000010:2.1:1787826430.631944:0:14640:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee800.
00000080:00000010:1.1:1787826430.685942:0:15993:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599440.
00010000:00000010:3.1:1787826430.700913:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87f43700.
00010000:00000010:3.1:1787826430.700938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefa00.
00000080:00000010:3.1:1787826430.715429:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:3.1:1787826430.715451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599340.
00010000:00000010:0.1:1787826430.759133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:0.1:1787826430.768817:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863999c0.
00010000:00000010:0.1:1787826430.786838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826430.786853:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399040.
00010000:00000010:0.1:1787826430.787984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:2.1:1787826430.831565:0:2341:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459240.
00010000:00000010:1.1:1787826430.847393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:1.1:1787826430.847412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826430.847421:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:1.1:1787826430.847435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599440.
00010000:00000010:1.1:1787826430.886704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4600.
00000080:00000010:1.1:1787826430.886715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399680.
00010000:00000010:1.1:1787826430.907710:0:15991:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fee200.
00000080:00000010:1.1:1787826430.907724:0:15991:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599440.
00010000:00000010:0.1:1787826430.921210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:2.1:1787826430.960397:0:15999:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:0.1:1787826430.988553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd99bda900.
00010000:00000010:0.1:1787826430.988565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288400.
00000080:00000010:0.1:1787826430.996717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:0.1:1787826430.996733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2480.
00010000:00000010:2.1:1787826431.055756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90289000.
00000080:00000010:2.1:1787826431.055778:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2140.
00010000:00000010:3.1:1787826431.081713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd90288000.
00000080:00000010:3.1:1787826431.081733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e2ac0.
00010000:00000010:1.1:1787826431.087065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:2.1:1787826431.149310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd905e23c0.
00010000:00000010:3.1:1787826431.187175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd87e97700.
00010000:00000010:3.1:1787826431.187198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c600.
00000080:00000010:1.1:1787826431.209263:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd845218c0.
00000080:00000010:1.1:1787826431.209281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87459660.
00010000:00000010:0.1:1787826431.226955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5800.
00000080:00000010:0.1:1787826431.226963:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd86399620.
00010000:00000010:0.1:1787826431.236833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd4800.
00000080:00000010:0.1:1787826431.236853:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd863993e0.
00010000:00000010:0.1:1787826431.252980:0:16003:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fd5a00.
00000080:00000010:1.1:1787826431.297927:0:16005:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599f20.
00010000:00000010:2.1:1787826431.320849:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707c00.
00010000:00000010:2.1:1787826431.320872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87feec00.
00000080:00000010:0.1:1787826431.336185:0:16006:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff95dd84515280.
00000080:00000010:0.1:1787826431.336220:0:16006:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd87599f20.
00000800:00000200:0.1:1787826458.876788:0:2338:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1
00000800:00000200:0.1:1787826458.876792:0:2338:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:0.1:1787826458.876794:0:2338:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000200:0.1:1787826472.701417:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826473.726487:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826473.726810:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826473.726971:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826477.757526:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826477.757844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826477.758128:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826477.758263:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826483.070105:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826483.070255:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826483.070409:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826483.070555:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826488.189870:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826488.189874:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 4, 1 secs later)
00000800:00000200:0.1:1787826488.190269:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826488.190272:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 4, 1 secs later)
00000800:00000200:0.1:1787826488.190566:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826488.190569:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 4, 1 secs later)
00000800:00000200:0.1:1787826488.190886:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826488.190889:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 4, 1 secs later)
00000800:00000200:3.1:1787826493.312508:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826493.313361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826493.313510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826493.313609:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826498.429640:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826498.429643:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 8, 4 secs later)
00000800:00000200:3.1:1787826498.430041:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826498.430044:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 8, 4 secs later)
00000800:00000200:3.1:1787826498.430358:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826498.430360:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 8, 4 secs later)
00000800:00000200:3.1:1787826498.430722:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826498.430725:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 8, 4 secs later)
00000800:00000200:3.1:1787826503.549332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826503.549708:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826503.549867:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826503.550010:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826508.670563:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826508.670576:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 12 secs later)
00000800:00000200:0.1:1787826508.671093:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826508.671095:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 12 secs later)
00000800:00000200:0.1:1787826508.671261:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826508.671262:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 12 secs later)
00000800:00000200:0.1:1787826508.671478:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826508.671480:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 12 secs later)
00000800:00000200:0.1:1787826513.789356:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826513.789362:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 7 secs later)
00000800:00000200:0.1:1787826513.789802:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826513.789814:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 7 secs later)
00000800:00000200:0.1:1787826513.789994:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826513.789996:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 7 secs later)
00000800:00000200:0.1:1787826513.790187:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826513.790188:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 7 secs later)
00000800:00000200:0.1:1787826518.910076:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826518.910080:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 2 secs later)
00000800:00000200:0.1:1787826518.910557:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826518.910559:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 2 secs later)
00000800:00000200:0.1:1787826518.910811:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826518.910813:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 2 secs later)
00000800:00000200:0.1:1787826518.911144:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826518.911145:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 16, 2 secs later)
00000800:00000200:0.1:1787826524.029592:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826524.030020:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826524.030159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826524.030305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826529.149303:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826529.149320:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 30 secs later)
00000800:00000200:3.1:1787826529.149887:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826529.149893:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 30 secs later)
00000800:00000200:3.1:1787826529.150243:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826529.150246:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 30 secs later)
00000800:00000200:3.1:1787826529.150484:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826529.150487:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 30 secs later)
00000800:00000200:3.1:1787826534.269232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826534.269238:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1787826534.269635:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826534.269637:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1787826534.269944:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826534.269946:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1787826534.270232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826534.270234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1787826539.389223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826539.389228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1787826539.389796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826539.389799:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1787826539.389999:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826539.390001:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1787826539.390166:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826539.390169:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1787826544.509523:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826544.509543:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1787826544.510101:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826544.510104:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1787826544.510399:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826544.510401:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1787826544.510720:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826544.510723:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1787826549.629313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826549.629319:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 9 secs later)
00000800:00000200:3.1:1787826549.629829:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826549.629832:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 9 secs later)
00000800:00000200:3.1:1787826549.630122:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826549.630125:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 9 secs later)
00000800:00000200:3.1:1787826549.630405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826549.630408:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 9 secs later)
00000800:00000200:3.1:1787826554.749324:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826554.749331:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 4 secs later)
00000800:00000200:3.1:1787826554.749817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826554.749820:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 4 secs later)
00000800:00000200:3.1:1787826554.750070:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826554.750073:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 4 secs later)
00000800:00000200:3.1:1787826554.750315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826554.750317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 32, 4 secs later)
00000800:00000200:3.1:1787826559.869258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826559.869644:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826559.869847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826559.869978:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826564.989595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826564.989598:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826564.989936:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826564.989939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826564.990206:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826564.990208:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826564.990490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826564.990492:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826570.109576:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826570.109582:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826570.109955:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826570.109957:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826570.110313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826570.110315:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826570.110578:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826570.110580:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826575.229439:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826575.229445:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787826575.230001:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826575.230003:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787826575.230287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826575.230289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787826575.230570:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826575.230573:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787826580.349990:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826580.349994:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787826580.350648:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826580.350651:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787826580.350973:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826580.350975:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787826580.351286:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826580.351289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787826585.469413:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826585.469418:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787826585.469979:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826585.469982:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787826585.470257:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826585.470260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787826585.470522:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826585.470525:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787826590.589649:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826590.589652:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787826590.589990:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826590.589993:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787826590.590287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826590.590290:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787826590.590535:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826590.590538:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787826595.709595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826595.709599:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787826595.709991:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826595.709994:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787826595.710283:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826595.710286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787826595.710702:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826595.710705:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787826600.829923:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826600.829930:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787826600.830551:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826600.830553:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787826600.830823:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826600.830825:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787826600.831159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826600.831161:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787826605.949507:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826605.949514:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787826605.950171:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826605.950173:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787826605.950374:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826605.950376:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787826605.950692:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826605.950695:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787826611.069631:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826611.069642:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787826611.070356:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826611.070359:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787826611.070693:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826611.070695:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787826611.070949:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826611.070951:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787826616.189411:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826616.189417:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787826616.189938:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826616.189940:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787826616.190212:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826616.190214:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787826616.190473:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826616.190476:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787826621.309616:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826621.309619:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787826621.309983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826621.309986:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787826621.310238:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826621.310240:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787826621.310518:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826621.310521:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787826626.429398:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826626.430092:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826626.430227:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826626.430351:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826631.550431:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826631.550433:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826631.550768:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826631.550770:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826631.551030:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826631.551032:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826631.551632:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826631.551634:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787826636.669875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826636.669879:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826636.670619:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826636.670622:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826636.670931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826636.670933:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826636.671204:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826636.671207:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787826641.790151:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826641.790154:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787826641.791958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826641.791975:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787826641.792271:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826641.792274:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787826641.792538:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826641.792540:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787826646.909346:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826646.909352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787826646.909719:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826646.909720:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787826646.909906:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826646.909907:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787826646.910061:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787826646.910063:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826652.029464:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826652.029472:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826652.030343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826652.030346:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826652.030646:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826652.030648:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826652.030970:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826652.030973:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826657.149329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826657.149335:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826657.149898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826657.149899:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826657.150170:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826657.150172:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826657.150427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826657.150429:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826662.269282:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826662.269286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826662.269817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826662.269819:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826662.269976:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826662.269977:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826662.270229:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826662.270232:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787826667.389524:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826667.389526:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787826667.389825:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826667.389826:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787826667.390047:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826667.390050:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787826667.390452:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826667.390454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787826672.510092:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826672.510096:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787826672.510442:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826672.510445:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787826672.510755:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826672.510757:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787826672.511034:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826672.511036:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787826677.629319:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826677.629326:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826677.629847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826677.629849:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826677.630048:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826677.630051:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826677.630221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826677.630222:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826682.749409:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826682.749414:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787826682.749920:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826682.749922:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787826682.750184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826682.750186:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787826682.750430:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826682.750432:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787826687.869333:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826687.869339:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787826687.869900:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826687.869903:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787826687.870168:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826687.870170:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787826687.870432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826687.870435:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787826692.989124:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826692.989317:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826692.989404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826692.989528:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826698.109388:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826698.109392:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826698.109963:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826698.109966:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826698.110208:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826698.110210:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826698.110487:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826698.110489:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787826703.229568:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826703.229572:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787826703.229950:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826703.229953:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787826703.230220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826703.230234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787826703.230651:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826703.230654:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787826708.349220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826708.349224:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787826708.349740:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826708.349742:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787826708.349960:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826708.349961:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787826708.350166:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826708.350168:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787826713.469572:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826713.469588:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787826713.469862:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826713.469864:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787826713.470127:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826713.470129:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787826713.470350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826713.470352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787826718.589593:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826718.589600:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787826718.590133:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826718.590136:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787826718.590361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826718.590362:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787826718.590596:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826718.590599:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787826723.709404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826723.709409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787826723.709922:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826723.709925:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787826723.710232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826723.710235:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787826723.710524:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826723.710527:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787826728.829250:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826728.829257:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787826728.829720:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826728.829722:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787826728.829915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826728.829916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787826728.830092:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826728.830093:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787826733.949362:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826733.949366:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787826733.949922:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826733.949924:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787826733.950213:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826733.950215:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787826733.950441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826733.950443:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787826739.069584:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826739.069591:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826739.070172:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826739.070174:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826739.070479:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826739.070481:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826739.070782:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826739.070785:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000080:00000010:2.1:1787826742.981702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff95dd92969560.
00010000:00000010:2.1:1787826742.981739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707700.
00010000:00000010:2.1:1787826742.981745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87f6c400.
00010000:00000010:2.1:1787826742.981748:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd91f4b800.
00010000:00000010:2.1:1787826742.981751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95ddc20fce00.
00000800:00000200:1.1:1787826744.189352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826744.189357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826744.189943:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826744.189946:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826744.190226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826744.190229:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826744.190427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826744.190429:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787826749.309936:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826749.309940:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787826749.310320:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826749.310323:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787826749.310642:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826749.310680:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787826749.310966:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787826749.310969:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787826754.429732:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826754.429739:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787826754.430329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826754.430333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787826754.430617:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826754.430620:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787826754.430932:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826754.430934:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787826759.549519:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826759.549937:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826759.550085:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826759.550214:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826764.669577:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826764.669588:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826764.670264:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826764.670267:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826764.670534:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826764.670537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826764.670817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826764.670820:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826769.789338:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826769.789345:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826769.789811:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826769.789813:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826769.790034:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826769.790036:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826769.790253:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826769.790254:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826774.909312:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826774.909317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826774.909915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826774.909917:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826774.910127:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826774.910129:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826774.910322:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826774.910324:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826780.029324:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826780.029331:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826780.029890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826780.029892:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826780.030104:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826780.030106:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826780.030261:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826780.030262:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826785.149537:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826785.149543:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826785.150187:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826785.150190:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826785.150480:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826785.150482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826785.150777:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826785.150780:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826790.270645:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826790.270648:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826790.271040:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826790.271042:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826790.271307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826790.271309:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826790.271570:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826790.271572:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826795.389793:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826795.389797:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826795.390203:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826795.390206:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826795.390531:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826795.390534:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826795.390844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826795.390847:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826800.509145:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826800.509148:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826800.509506:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826800.509509:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826800.509747:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826800.509749:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826800.509911:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826800.509912:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826805.629584:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826805.629586:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826805.630070:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826805.630072:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826805.630334:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826805.630336:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826805.630596:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826805.630598:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826810.749348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826810.749353:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787826810.749890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826810.749893:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787826810.750156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826810.750158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787826810.750416:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826810.750418:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787826815.869902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826815.869912:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787826815.870768:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826815.870770:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787826815.870962:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826815.870964:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787826815.871234:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826815.871237:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787826820.989846:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826820.989850:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787826820.990481:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826820.990484:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787826820.990814:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826820.990817:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787826820.991368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826820.991370:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787826826.109294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826826.109629:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826826.109802:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826826.109961:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826831.229599:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826831.229602:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787826831.229962:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826831.229964:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787826831.230220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826831.230222:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787826831.230516:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826831.230518:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787826836.349640:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826836.349645:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787826836.350253:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826836.350256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787826836.350555:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826836.350558:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787826836.350874:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826836.350876:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787826841.469605:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826841.469611:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787826841.470089:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826841.470091:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787826841.470371:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826841.470386:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787826841.471449:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826841.471451:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787826846.590254:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826846.590258:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787826846.590619:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826846.590622:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787826846.590914:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826846.590916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787826846.591170:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826846.591172:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787826851.709283:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826851.709289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826851.709715:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826851.709718:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826851.710547:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826851.710550:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826851.710900:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826851.710902:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826856.829403:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826856.829409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826856.830045:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826856.830046:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826856.830251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826856.830264:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826856.830445:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826856.830447:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826861.949251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826861.949269:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826861.949876:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826861.949878:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826861.950148:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826861.950150:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826861.950404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826861.950406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826867.069400:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826867.069406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826867.069898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826867.069901:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826867.070143:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826867.070146:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826867.070390:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826867.070392:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826872.190330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826872.190337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826872.190853:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826872.190856:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826872.191143:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826872.191157:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826872.191437:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826872.191439:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787826877.309379:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826877.309385:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787826877.309916:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826877.309918:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787826877.310207:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826877.310209:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787826877.310468:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826877.310470:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787826882.429348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826882.429354:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787826882.429920:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826882.429922:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787826882.430193:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826882.430196:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787826882.430413:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826882.430415:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787826887.549518:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826887.549524:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787826887.550087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826887.550090:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787826887.550355:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826887.550370:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787826887.550627:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826887.550629:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787826892.669310:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826892.669697:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826892.669852:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826892.669975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826897.791646:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826897.791649:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826897.792057:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826897.792060:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826897.792351:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826897.792353:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826897.792630:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826897.792632:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826902.909321:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826902.909326:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826902.909882:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826902.909885:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826902.910136:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826902.910138:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826902.910311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826902.910312:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826908.029323:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826908.029330:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826908.029966:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826908.029968:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826908.030201:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826908.030203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826908.030422:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826908.030424:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826913.149259:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826913.149266:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826913.149862:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826913.149864:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826913.150096:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826913.150098:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826913.150289:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826913.150291:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826918.269431:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826918.269440:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826918.270315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826918.270317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826918.270573:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826918.270576:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826918.270875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826918.270878:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787826923.389346:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826923.389352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826923.389901:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826923.389902:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826923.390138:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826923.390140:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826923.390365:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826923.390367:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787826928.509295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826928.509299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826928.509811:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826928.509813:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826928.510033:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826928.510035:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826928.510265:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826928.510268:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787826933.629225:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826933.629228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826933.629608:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826933.629610:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826933.629945:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826933.629947:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787826933.630210:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826933.630213:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787826938.749402:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826938.749405:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826938.750001:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826938.750003:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826938.750264:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826938.750278:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826938.750537:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826938.750540:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787826943.869529:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826943.869533:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787826943.869912:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826943.869915:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787826943.870219:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826943.870222:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787826943.870612:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826943.870614:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787826948.989639:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826948.989642:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787826948.990039:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826948.990042:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787826948.990325:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826948.990328:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787826948.990595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826948.990597:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787826954.109506:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826954.109509:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787826954.109825:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826954.109828:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787826954.110055:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826954.110067:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787826954.110323:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826954.110325:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787826959.229440:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826959.229818:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826959.229969:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826959.231186:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826964.350369:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826964.350374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787826964.350821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826964.350823:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787826964.351098:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826964.351101:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787826964.351369:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826964.351371:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787826969.469102:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826969.469106:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826969.469471:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826969.469473:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826969.469712:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826969.469715:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826969.469962:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826969.469965:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787826974.589352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826974.589357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826974.589937:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826974.589940:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826974.590200:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826974.590203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787826974.590486:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826974.590489:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00010000:00000010:0.1:1787826978.498719:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff95dd92707b00.
00010000:00000010:0.1:1787826978.498734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff95dd87fefe00.
00000800:00000200:0.1:1787826979.709350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826979.709355:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826979.709883:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826979.709885:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826979.710150:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826979.710153:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826979.710399:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826979.710402:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787826984.829316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826984.829331:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826984.830009:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826984.830012:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826984.830294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826984.830296:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826984.830642:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826984.830644:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787826989.949275:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826989.949280:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826989.949860:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826989.949862:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826989.950109:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826989.950111:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787826989.950363:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787826989.950379:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787826995.069647:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826995.069651:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787826995.070774:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826995.070777:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787826995.071078:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826995.071081:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787826995.071495:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787826995.071497:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827000.189419:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827000.189424:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827000.190007:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827000.190009:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827000.190284:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827000.190287:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827000.190561:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827000.190563:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827005.309446:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827005.309451:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827005.310015:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827005.310018:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827005.310305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827005.310308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827005.310573:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827005.310576:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827010.429689:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827010.429693:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827010.430008:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827010.430011:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827010.430294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827010.430297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827010.430559:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827010.430561:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827015.549417:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827015.549423:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827015.550073:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827015.550075:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827015.550343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827015.550346:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827015.550678:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827015.550680:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827020.669379:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827020.669397:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827020.669947:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827020.669949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827020.670232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827020.670235:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827020.670499:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827020.670502:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827025.789303:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827025.789699:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827025.789826:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827025.789985:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827030.909401:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827030.909406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827030.910000:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827030.910003:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827030.910294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827030.910297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827030.910559:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827030.910562:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827036.029235:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827036.029242:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827036.029771:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827036.029774:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827036.030036:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827036.030039:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827036.030327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827036.030329:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827041.149220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827041.149225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787827041.149628:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827041.149630:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787827041.149828:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827041.149830:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787827041.150022:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827041.150025:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787827046.269395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827046.269403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827046.270016:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827046.270019:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827046.270262:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827046.270264:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827046.270475:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827046.270478:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827051.389315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827051.389322:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827051.389930:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827051.389933:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827051.390205:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827051.390208:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827051.390496:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827051.390499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827056.509557:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827056.509567:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827056.510414:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827056.510417:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827056.510728:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827056.510730:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827056.511028:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827056.511030:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827061.629637:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827061.629644:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827061.630136:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827061.630139:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827061.630413:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827061.630415:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827061.630719:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827061.630722:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827066.750085:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827066.750088:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827066.750414:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827066.750416:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827066.750699:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827066.750701:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827066.750975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827066.750977:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827071.869729:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827071.869735:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827071.870264:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827071.870266:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827071.870512:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827071.870513:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827071.870736:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827071.870737:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827076.989380:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827076.989385:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827076.989824:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827076.989826:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827076.990069:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827076.990070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827076.990267:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827076.990269:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827082.109325:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827082.109330:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827082.109828:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827082.109830:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827082.110104:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827082.110107:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827082.110377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827082.110380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827087.229259:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827087.229264:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827087.229827:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827087.229829:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827087.230036:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827087.230037:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827087.230295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827087.230297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787827092.349215:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827092.349509:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827092.349815:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827092.349973:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827097.469435:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827097.469439:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827097.469888:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827097.469903:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827097.470247:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827097.470249:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827097.470513:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827097.470516:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827102.589315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827102.589320:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827102.589902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827102.589905:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827102.590179:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827102.590182:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827102.590439:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827102.590441:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827107.709569:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827107.709578:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827107.710273:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827107.710276:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827107.710546:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827107.710558:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827107.710823:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827107.710826:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787827112.829289:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827112.829292:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827112.829869:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827112.829871:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827112.830052:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827112.830054:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827112.830362:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827112.830364:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827117.949396:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827117.949400:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827117.949965:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827117.949968:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827117.950307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827117.950309:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827117.950510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827117.950513:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827123.069392:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827123.069395:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827123.070593:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827123.070612:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827123.070968:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827123.070971:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827123.071236:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827123.071239:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827128.189316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827128.189321:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787827128.189757:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827128.189759:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787827128.190012:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827128.190014:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787827128.190178:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827128.190179:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787827133.309700:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827133.309707:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827133.310209:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827133.310212:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827133.310496:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827133.310499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827133.310781:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827133.310784:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827138.429283:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827138.429288:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827138.429890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827138.429905:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827138.430161:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827138.430163:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827138.430453:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827138.430454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827143.549394:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827143.549400:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827143.549934:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827143.549936:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827143.550209:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827143.550211:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827143.550498:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827143.550500:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827148.669405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827148.669410:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827148.669993:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827148.669995:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827148.670287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827148.670290:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827148.670559:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827148.670561:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827153.789351:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827153.789356:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827153.789931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827153.789946:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827153.790198:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827153.790200:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827153.790550:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827153.790553:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827158.909540:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827158.909796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827158.909940:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827158.910117:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827164.029435:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827164.029442:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827164.029900:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827164.029903:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827164.030199:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827164.030201:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827164.030469:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827164.030471:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827169.149350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827169.149357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827169.149877:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827169.149880:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827169.150172:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827169.150175:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827169.150451:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827169.150453:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827174.269326:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827174.269332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827174.269901:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827174.269904:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827174.270195:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827174.270198:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827174.270479:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827174.270496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787827179.389217:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827179.389222:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827179.389651:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827179.389652:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827179.389858:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827179.389860:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827179.390018:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827179.390020:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787827184.509226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827184.509232:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827184.509594:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827184.509606:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827184.509801:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827184.509803:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827184.509954:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827184.509956:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787827189.629357:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827189.629365:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827189.629888:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827189.629891:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827189.630242:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827189.630245:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827189.630573:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827189.630575:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787827194.749317:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827194.749322:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827194.749810:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827194.749812:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827194.750024:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827194.750026:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827194.750203:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827194.750205:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827199.869318:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827199.869325:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827199.869959:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827199.869962:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827199.870215:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827199.870217:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827199.870444:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827199.870446:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827204.989318:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827204.989323:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827204.989938:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827204.989943:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827204.990231:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827204.990233:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827204.990415:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827204.990417:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827210.109401:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827210.109406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827210.109953:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827210.109974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827210.110223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827210.110225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827210.110417:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827210.110419:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827215.229383:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827215.229390:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827215.229866:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827215.229867:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827215.230096:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827215.230097:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827215.230312:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827215.230330:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827220.349316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827220.349323:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827220.349837:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827220.349839:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827220.350059:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827220.350060:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827220.350266:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827220.350268:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827225.469441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827225.469863:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827225.470045:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827225.470196:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827230.589595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827230.589604:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827230.590349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827230.590352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827230.590577:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827230.590579:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827230.590856:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827230.590858:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827235.709466:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827235.709475:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827235.710253:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827235.710255:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827235.710599:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827235.710602:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787827235.710974:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827235.710977:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827240.829354:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827240.829362:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827240.829893:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827240.829895:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827240.830159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827240.830161:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827240.830375:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827240.830377:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787827245.949355:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827245.949361:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827245.949931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827245.949934:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827245.950241:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827245.950244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827245.950534:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827245.950537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787827251.069124:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827251.069128:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827251.069491:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827251.069504:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827251.069731:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827251.069732:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827251.069910:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827251.069911:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787827256.189464:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827256.189471:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827256.190156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827256.190159:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827256.190480:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827256.190482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827256.190841:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827256.190844:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787827261.309571:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827261.309580:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827261.310165:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827261.310167:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827261.310362:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827261.310365:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827261.310689:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827261.310691:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787827266.429510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827266.429518:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827266.429949:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827266.429968:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827266.430175:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827266.430178:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827266.430418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827266.430420:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827271.549253:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827271.549261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827271.549817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827271.549819:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827271.550005:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827271.550006:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827271.550216:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827271.550234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827276.669418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827276.669429:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827276.669984:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827276.669986:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827276.670205:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827276.670207:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827276.670434:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827276.670436:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787827281.789414:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827281.789422:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827281.789839:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827281.789841:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827281.790027:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827281.790028:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827281.790199:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827281.790200:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787827286.909350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827286.909357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827286.909854:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827286.909856:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827286.910181:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827286.910184:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787827286.910498:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827286.910501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827292.029690:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827292.030294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827292.030510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827292.030730:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827297.149532:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827297.149557:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827297.150350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827297.150354:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827297.150694:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827297.150697:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827297.150914:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827297.150916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787827302.269481:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827302.269489:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787827302.270367:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827302.270380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787827302.270571:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827302.270573:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787827302.270774:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827302.270775:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787827307.389299:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827307.389303:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787827307.389776:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827307.389777:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787827307.389997:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827307.389999:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787827307.390192:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827307.390193:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827312.509331:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827312.509336:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827312.509929:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827312.509932:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827312.510272:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827312.510275:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827312.510591:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827312.510594:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827317.629439:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827317.629444:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827317.629977:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827317.629979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827317.630202:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827317.630203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827317.630423:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827317.630425:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827322.749308:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827322.749330:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827322.749801:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827322.749803:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827322.749989:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827322.749991:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827322.750189:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827322.750190:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827327.869246:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827327.869250:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827327.869735:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827327.869737:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827327.869924:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827327.869925:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827327.870136:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827327.870154:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827332.989382:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827332.989387:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827332.989916:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827332.989919:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827332.990233:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827332.990236:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827332.990468:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827332.990470:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787827338.109341:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827338.109352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827338.109785:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827338.109787:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827338.110013:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827338.110015:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787827338.110301:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787827338.110303:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827343.229383:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827343.229401:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787827343.229996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827343.229998:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787827343.230190:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827343.230192:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787827343.230407:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827343.230409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787827348.349532:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827348.349536:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827348.350025:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827348.350027:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827348.350305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827348.350307:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827348.350537:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827348.350539:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827353.469159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827353.469163:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827353.469506:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827353.469519:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827353.469733:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827353.469734:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827353.469963:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827353.469966:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827358.589354:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827358.589817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827358.590019:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827358.590177:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827363.709758:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827363.709766:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827363.710357:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827363.710360:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827363.710688:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827363.710691:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827363.711039:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827363.711042:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827368.829951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827368.829960:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827368.830654:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827368.830657:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827368.831076:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827368.831080:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827368.831506:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827368.831510:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827373.949553:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827373.949558:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827373.950236:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827373.950238:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827373.950623:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827373.950625:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827373.951030:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827373.951032:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827379.069346:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827379.069351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827379.070061:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827379.070063:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827379.070298:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827379.070300:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827379.070565:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827379.070568:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827384.189617:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827384.189641:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827384.190338:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827384.190340:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827384.190714:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827384.190728:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827384.191063:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827384.191066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827389.309569:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827389.309615:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827389.310535:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827389.310538:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827389.310976:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827389.310978:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827389.311280:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827389.311283:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827394.429367:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827394.429373:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827394.430058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827394.430061:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827394.430298:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827394.430299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827394.430530:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827394.430532:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827399.549274:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827399.549280:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827399.549937:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827399.549939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827399.550194:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827399.550196:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827399.550424:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827399.550425:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827404.669243:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827404.669247:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827404.669863:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827404.669865:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827404.670054:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827404.670056:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827404.670293:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827404.670295:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827409.789589:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827409.789615:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827409.790477:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827409.790479:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827409.790822:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827409.790825:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827409.791201:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827409.791203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827414.909907:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827414.909919:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827414.911045:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827414.911048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827414.911451:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827414.911454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827414.911901:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827414.911904:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827420.029515:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827420.029523:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827420.030124:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827420.030128:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827420.030469:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827420.030471:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827420.030754:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827420.030756:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827425.149246:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827425.149826:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827425.150036:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827425.150184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827430.269595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827430.269604:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827430.270217:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827430.270220:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827430.270963:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827430.270966:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827430.271223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827430.271225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827435.389338:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827435.389343:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827435.390043:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827435.390047:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827435.390300:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827435.390301:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827435.390579:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827435.390580:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827440.509258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827440.509264:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827440.509706:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827440.509708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827440.509978:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827440.509979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827440.510192:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827440.510193:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827445.629430:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827445.629437:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827445.629958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827445.629960:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827445.630255:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827445.630256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827445.630460:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827445.630461:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827450.749595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827450.749607:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827450.750271:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827450.750272:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827450.750488:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827450.750489:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827450.750718:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827450.750721:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827455.869384:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827455.869387:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787827455.870022:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827455.870024:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787827455.870316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827455.870318:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787827455.870560:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827455.870561:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787827460.989307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827460.989313:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827460.989926:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827460.989928:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827460.990130:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827460.990131:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827460.990359:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827460.990361:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827466.109395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827466.109401:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827466.109962:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827466.109964:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827466.110189:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827466.110191:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827466.110480:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827466.110482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787827471.229463:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827471.229466:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827471.230398:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827471.230400:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827471.230623:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827471.230626:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827471.230979:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827471.230981:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827476.349516:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827476.349522:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827476.350087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827476.350089:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827476.350336:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827476.350339:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827476.350592:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827476.350593:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827481.469313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827481.469320:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827481.469868:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827481.469870:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827481.470105:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827481.470107:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827481.470294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827481.470296:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827486.589344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827486.589349:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827486.589762:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827486.589764:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827486.589976:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827486.589978:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827486.590162:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827486.590164:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827491.709437:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827491.709871:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827491.710001:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827491.710175:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827496.829707:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827496.829716:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827496.830424:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827496.830428:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827496.830836:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827496.830877:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827496.831198:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827496.831200:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827501.949319:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827501.949326:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827501.949931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827501.949933:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827501.950116:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827501.950118:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827501.950349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827501.950351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827507.069494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827507.069499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827507.070187:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827507.070190:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827507.070512:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827507.070515:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827507.070863:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827507.070866:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827512.189284:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827512.189291:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827512.189757:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827512.189759:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827512.189947:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827512.189949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827512.190142:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827512.190144:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827517.309403:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827517.309409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827517.309899:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827517.309901:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827517.310128:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827517.310130:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827517.310314:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827517.310316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827522.429243:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827522.429249:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827522.429705:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827522.429707:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827522.429905:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827522.429908:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827522.430104:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827522.430106:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827527.549484:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827527.549490:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827527.549924:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827527.549926:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827527.550147:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827527.550149:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827527.550390:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827527.550392:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827532.669680:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827532.669689:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827532.670273:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827532.670276:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827532.670611:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827532.670615:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827532.670854:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827532.670856:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827537.789627:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827537.789637:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827537.790436:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827537.790439:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827537.790815:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827537.790817:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827537.791104:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827537.791107:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827542.909632:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827542.909640:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827542.910552:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827542.910555:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827542.910888:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827542.910891:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827542.911165:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827542.911167:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827548.029327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827548.029332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827548.029918:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827548.029920:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827548.030229:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827548.030230:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827548.030511:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827548.030513:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827553.149284:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827553.149289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827553.149855:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827553.149857:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827553.150059:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827553.150062:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787827553.150286:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827553.150287:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827558.269377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827558.269716:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827558.269863:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827558.270056:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827563.389737:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827563.389747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827563.390510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827563.390519:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827563.391540:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827563.391543:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827563.391885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827563.391888:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827568.509475:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827568.509483:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827568.510101:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827568.510103:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827568.510386:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827568.510417:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827568.510748:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827568.510750:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827573.629578:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827573.629590:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827573.630352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827573.630354:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827573.630632:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827573.630634:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827573.630873:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827573.630875:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827578.749368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827578.749378:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827578.750002:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827578.750016:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827578.750250:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827578.750252:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827578.750585:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827578.750587:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827583.869454:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827583.869465:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827583.870054:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827583.870058:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827583.870431:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827583.870435:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827583.870865:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827583.870868:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827588.989627:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827588.989638:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787827588.990329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827588.990331:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787827588.990530:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827588.990532:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787827588.990750:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827588.990752:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787827594.109602:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827594.109610:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827594.110422:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827594.110425:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827594.110741:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827594.110744:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827594.111058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827594.111061:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787827599.229868:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827599.229876:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827599.230820:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827599.230846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827599.231370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827599.231373:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827599.231739:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827599.231742:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827604.349260:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827604.349265:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827604.349753:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827604.349756:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827604.349975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827604.349978:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827604.350189:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827604.350191:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827609.469948:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827609.469958:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827609.471358:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827609.471363:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827609.471910:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827609.471913:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827609.472370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827609.472374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827614.589332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827614.589336:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827614.589818:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827614.589820:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827614.590019:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827614.590020:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827614.590237:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827614.590238:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827619.709289:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827619.709293:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827619.709885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827619.709899:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827619.710131:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827619.710134:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827619.710339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827619.710341:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827624.830188:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827624.830878:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827624.831122:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827624.831343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827629.950189:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827629.950207:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827629.951482:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827629.951486:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827629.952099:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827629.952105:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827629.952849:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827629.952854:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827635.069680:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827635.069685:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827635.070190:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827635.070192:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827635.070372:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827635.070374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827635.070586:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827635.070588:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827640.189740:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827640.189752:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827640.190718:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827640.190721:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827640.190937:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827640.190939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827640.191155:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827640.191157:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827645.309381:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827645.309391:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827645.309792:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827645.309794:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827645.310009:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827645.310011:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827645.310197:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827645.310199:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827650.429213:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827650.429220:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827650.429718:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827650.429720:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827650.429923:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827650.429925:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827650.430097:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827650.430099:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827655.549581:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827655.549593:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827655.550360:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827655.550362:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827655.550544:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827655.550546:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827655.550756:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827655.550758:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827660.669277:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827660.669286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827660.669881:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827660.669883:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827660.670083:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827660.670085:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827660.670271:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827660.670273:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827665.789215:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827665.789219:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827665.789749:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827665.789750:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827665.789937:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827665.789939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827665.790108:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827665.790110:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827670.909329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827670.909334:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827670.909799:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827670.909802:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827670.910119:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827670.910121:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827670.910427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827670.910431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827676.029404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827676.029412:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827676.029862:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827676.029864:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827676.030064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827676.030066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827676.030303:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827676.030306:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827681.149279:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827681.149284:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827681.149830:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827681.149832:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827681.150064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827681.150066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827681.150263:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827681.150265:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827686.269394:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827686.269404:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827686.269860:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827686.269862:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827686.270109:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827686.270111:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827686.270359:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827686.270361:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827691.389214:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827691.389529:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827691.389704:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827691.389875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827696.509790:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827696.509795:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827696.510811:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827696.510814:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827696.511170:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827696.511173:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827696.511513:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827696.511516:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827701.629498:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827701.629506:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827701.630242:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827701.630244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827701.630499:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827701.630513:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827701.630720:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827701.630722:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827706.749617:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827706.749628:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827706.750212:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827706.750214:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827706.750515:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827706.750517:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827706.750825:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827706.750828:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827711.869662:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827711.869666:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827711.870343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827711.870383:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827711.870806:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827711.870809:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827711.871156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827711.871159:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827716.989224:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827716.989231:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827716.989707:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827716.989708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827716.989902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827716.989915:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827716.990100:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827716.990102:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827722.109342:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827722.109348:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787827722.109915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827722.109916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787827722.110171:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827722.110190:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787827722.110441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827722.110443:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787827727.229594:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827727.229604:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827727.230472:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827727.230474:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827727.230852:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827727.230854:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827727.231109:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827727.231111:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827732.349256:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827732.349274:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827732.349942:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827732.349944:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827732.350154:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827732.350155:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827732.350355:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827732.350357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827737.469800:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827737.469812:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827737.470679:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827737.470681:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827737.470914:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827737.470916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827737.471215:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827737.471217:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827742.589181:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827742.589186:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827742.589728:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827742.589730:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827742.589914:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827742.589916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827742.590087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827742.590088:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787827747.709372:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827747.709378:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827747.710052:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827747.710053:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827747.710304:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827747.710318:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787827747.710569:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827747.710571:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827752.829628:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827752.829635:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827752.830450:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827752.830453:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827752.830685:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827752.830699:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827752.830943:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827752.830945:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827757.949356:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827757.949688:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827757.949796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827757.949931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827763.069350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827763.069357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827763.070004:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827763.070008:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827763.070769:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827763.070773:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787827763.070977:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827763.070979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827768.189398:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827768.189403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827768.189989:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827768.189990:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827768.190179:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827768.190181:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827768.190446:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827768.190448:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827773.309316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827773.309321:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827773.309816:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827773.309818:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827773.310057:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827773.310062:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827773.310243:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827773.310244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787827778.429345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827778.429351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827778.429821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827778.429823:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827778.430071:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827778.430073:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827778.430314:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827778.430317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787827783.549641:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827783.549652:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827783.550203:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827783.550207:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827783.550452:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827783.550454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827783.550708:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827783.550711:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827788.669218:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827788.669226:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827788.669785:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827788.669787:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827788.670005:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827788.670007:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827788.670202:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827788.670204:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827793.789339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827793.789344:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827793.790009:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827793.790011:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827793.790230:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827793.790232:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827793.790461:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827793.790463:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827798.909772:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827798.909776:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827798.910383:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827798.910386:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827798.910808:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827798.910811:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827798.911142:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827798.911145:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827804.029902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827804.029910:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827804.030619:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827804.030623:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827804.030940:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827804.030943:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827804.031251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827804.031253:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827809.150051:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827809.150067:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827809.150904:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827809.150908:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827809.151381:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827809.151387:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827809.151880:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827809.151883:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827814.269974:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827814.269987:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827814.271291:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827814.271297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827814.271747:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827814.271750:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827814.272384:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827814.272388:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787827819.389911:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827819.389922:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827819.390798:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827819.390802:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827819.391159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827819.391162:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827819.391518:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827819.391521:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787827824.509822:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827824.510153:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827824.510271:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827824.510378:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827829.629616:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827829.629621:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827829.630457:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827829.630459:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827829.630694:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827829.630696:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827829.630890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827829.630907:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827834.749276:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827834.749281:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827834.749703:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827834.749704:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827834.749892:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827834.749894:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827834.750135:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827834.750138:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827839.869628:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827839.869639:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827839.870820:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827839.870823:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827839.871038:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827839.871040:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827839.871327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827839.871329:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787827844.989638:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827844.989653:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827844.990718:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827844.990720:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827844.991011:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827844.991013:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827844.991377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827844.991396:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787827850.109646:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827850.109703:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827850.110803:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827850.110807:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827850.111231:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827850.111235:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787827850.111711:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827850.111714:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787827855.229617:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827855.229624:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827855.230465:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827855.230469:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827855.230764:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827855.230779:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827855.231032:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827855.231034:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827860.349686:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827860.349690:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827860.350388:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827860.350391:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827860.350603:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827860.350615:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827860.350931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827860.350933:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827865.469495:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827865.469502:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827865.470093:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827865.470095:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827865.470339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827865.470341:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827865.470609:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827865.470611:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827870.589966:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827870.589975:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827870.591025:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827870.591028:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827870.591539:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827870.591568:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827870.592067:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827870.592070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787827875.709891:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827875.709899:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827875.711045:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827875.711048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827875.711470:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827875.711473:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827875.711905:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827875.711908:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787827880.829317:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827880.829323:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827880.829889:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827880.829891:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827880.830104:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827880.830107:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827880.830369:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827880.830390:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787827885.949368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827885.949380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827885.950184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827885.950187:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827885.950432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827885.950433:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827885.950689:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827885.950691:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787827891.069541:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827891.069965:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827891.070143:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827891.070306:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827896.189456:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827896.189462:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827896.190233:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827896.190236:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827896.190566:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827896.190569:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827896.190865:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827896.190867:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827901.309515:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827901.309528:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787827901.310224:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827901.310227:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787827901.310492:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827901.310494:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787827901.310758:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827901.310759:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787827906.429494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827906.429501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787827906.430112:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827906.430114:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787827906.430385:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827906.430388:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787827906.430622:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827906.430624:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787827911.549226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827911.549231:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787827911.549769:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827911.549771:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787827911.550003:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827911.550005:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787827911.550186:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827911.550188:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787827916.669434:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827916.669441:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827916.669891:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827916.669893:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827916.670149:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827916.670166:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827916.670361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827916.670363:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827921.789319:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827921.789323:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827921.789728:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827921.789730:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827921.789979:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827921.789981:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827921.790195:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827921.790196:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827926.909997:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827926.910017:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827926.911630:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827926.911635:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827926.912357:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827926.912363:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827926.913073:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827926.913079:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827932.029328:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827932.029333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827932.029775:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827932.029777:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827932.029988:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827932.030003:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827932.030183:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827932.030185:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787827937.149400:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827937.149405:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827937.150133:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827937.150136:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827937.150358:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827937.150360:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827937.150595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827937.150597:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787827942.269459:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827942.269464:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787827942.270232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827942.270235:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787827942.270644:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827942.270647:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787827942.271145:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827942.271148:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787827947.389310:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827947.389316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827947.389744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827947.389746:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827947.389938:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827947.389954:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827947.390136:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827947.390137:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787827952.509921:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827952.509931:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827952.511437:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827952.511441:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827952.511808:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827952.511811:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827952.512142:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827952.512144:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787827957.629820:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827957.630471:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827957.630958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827957.631276:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827962.750018:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827962.750030:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827962.751574:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827962.751580:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827962.752295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827962.752300:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827962.753205:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827962.753211:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787827967.869916:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827967.869927:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827967.870764:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827967.870767:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827967.871125:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827967.871129:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787827967.871533:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827967.871537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787827972.990020:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827972.990035:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827972.991050:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827972.991053:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827972.991479:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827972.991482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827972.991951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827972.991954:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787827978.109940:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827978.109949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827978.111043:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827978.111046:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827978.111777:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827978.111781:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827978.112213:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827978.112218:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787827983.229996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827983.230008:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827983.231117:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827983.231123:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827983.231883:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827983.231888:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787827983.232565:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827983.232571:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787827988.349544:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827988.349550:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827988.350281:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827988.350285:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827988.350741:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827988.350744:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787827988.351098:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827988.351101:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787827993.469571:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827993.469580:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827993.470510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827993.470513:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827993.470836:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827993.470838:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787827993.471175:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787827993.471177:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787827998.589616:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827998.589621:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827998.590219:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827998.590222:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827998.590527:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827998.590529:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787827998.590787:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787827998.590789:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787828003.709235:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828003.709242:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828003.709718:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828003.709720:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828003.709930:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828003.709931:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828003.710127:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828003.710129:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828008.829803:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828008.829819:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828008.831225:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828008.831229:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828008.831666:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828008.831669:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828008.832330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828008.832332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828013.949603:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828013.949611:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828013.950379:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828013.950383:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828013.950776:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828013.950779:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828013.951123:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828013.951126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828019.069340:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828019.069346:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828019.069798:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828019.069800:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828019.070006:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828019.070008:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828019.070186:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828019.070188:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828024.189386:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828024.189723:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828024.189893:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828024.190060:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828029.309770:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828029.309776:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828029.310595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828029.310600:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828029.311087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828029.311090:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828029.311480:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828029.311484:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828034.429996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828034.430004:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828034.431087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828034.431091:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828034.431588:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828034.431591:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828034.431953:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828034.431956:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828039.549921:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828039.549941:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828039.551305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828039.551311:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828039.551785:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828039.551789:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828039.552507:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828039.552532:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828044.669650:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828044.669661:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828044.670474:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828044.670479:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828044.670848:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828044.670850:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828044.671152:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828044.671153:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828049.789382:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828049.789386:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828049.789982:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828049.789985:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828049.790283:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828049.790285:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828049.790499:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828049.790501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828054.909893:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828054.909906:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828054.910877:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828054.910883:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828054.911315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828054.911318:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828054.911747:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828054.911767:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828060.030337:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828060.030354:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828060.031235:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828060.031237:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828060.031624:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828060.031627:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828060.032035:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828060.032037:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828065.149875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828065.149884:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787828065.150884:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828065.150888:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787828065.151318:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828065.151321:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787828065.151752:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828065.151755:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787828070.269742:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828070.269750:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828070.270929:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828070.270934:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828070.271385:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828070.271389:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828070.271848:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828070.271852:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828075.389933:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828075.389939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828075.390777:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828075.390783:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828075.391206:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828075.391209:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828075.391748:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828075.391774:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828080.510032:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828080.510041:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828080.511179:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828080.511182:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828080.511564:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828080.511568:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828080.511988:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828080.511991:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787828085.630006:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828085.630015:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828085.630902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828085.630905:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828085.631321:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828085.631324:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828085.631705:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828085.631708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828090.749878:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828090.750635:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828090.751086:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828090.751404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828095.870043:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828095.870052:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828095.871246:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828095.871251:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828095.871886:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828095.871891:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828095.872351:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828095.872356:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828100.989893:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828100.989923:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787828100.990881:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828100.990884:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787828100.991247:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828100.991250:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787828100.991615:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828100.991618:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787828106.109717:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828106.109724:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828106.110411:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828106.110414:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828106.110842:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828106.110845:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828106.111187:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828106.111191:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828111.230020:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828111.230037:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828111.231544:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828111.231547:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828111.231984:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828111.231987:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828111.232432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828111.232435:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828116.350303:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828116.350350:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828116.351236:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828116.351240:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828116.351756:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828116.351760:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828116.352296:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828116.352301:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828121.470074:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828121.470082:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787828121.471304:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828121.471308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787828121.471888:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828121.471894:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787828121.472309:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828121.472313:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828126.589847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828126.589858:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828126.590958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828126.590961:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828126.591164:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828126.591166:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828126.591366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828126.591368:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787828131.709641:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828131.709649:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787828131.710405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828131.710407:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787828131.710715:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828131.710717:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787828131.711061:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828131.711064:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787828136.829943:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828136.829978:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828136.831096:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828136.831099:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828136.831524:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828136.831527:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828136.831933:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828136.831937:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828141.949860:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828141.949868:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828141.950781:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828141.950785:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828141.951256:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828141.951261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828141.951692:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828141.951714:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828147.069796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828147.069805:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828147.070821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828147.070825:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828147.071193:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828147.071197:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828147.071590:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828147.071593:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828152.189811:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828152.189816:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828152.190897:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828152.190901:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828152.191366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828152.191369:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828152.191755:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828152.191759:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828157.309348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828157.309616:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828157.309774:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828157.309924:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828162.429970:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828162.429979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828162.430823:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828162.430826:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828162.431185:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828162.431188:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828162.431579:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828162.431601:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828167.549939:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828167.549955:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828167.551219:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828167.551223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828167.551462:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828167.551464:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828167.551855:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828167.551858:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787828172.669602:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828172.669615:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828172.670776:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828172.670779:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828172.671070:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828172.671073:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787828172.671445:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828172.671448:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828177.789833:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828177.789848:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828177.790744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828177.790747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828177.791220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828177.791224:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828177.791641:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828177.791644:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787828182.910181:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828182.910198:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787828182.911409:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828182.911412:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787828182.911810:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828182.911812:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787828182.912221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828182.912223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787828188.029544:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828188.029552:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787828188.030100:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828188.030101:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787828188.030295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828188.030297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787828188.030494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828188.030495:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787828193.149304:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828193.149309:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787828193.149791:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828193.149793:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000100:00000001:2.0:1787828193.149862:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828193.149865:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828193.149866:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828193.149871:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828193/real 1787828193]  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828193.149880:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828193.149882:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828193.149884:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828193.149887:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828193.149888:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828193.149891:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828193.149894:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828193.149900:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.149904:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828193.149905:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.149906:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828193.149907:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828193.149909:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.149914:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828193.149916:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.149921:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828193.149922:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.149923:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828193.149924:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.149925:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828193.149927:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828193.149928:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828193.149931:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828193.149933:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828193.149936:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828193.149941:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327219712:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828193.149943:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828193.149945:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.149949:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828193.149950:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828193.149951:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828193.149952:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828193.149954:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828193.149956:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327219840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828193.149958:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828193.149959:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828193.149960:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828193.149962:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828193.149964:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828193.149966:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828193.149968:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828193.149970:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828193.149972:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828193.149975:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327219840, portal 4
00000100:00000001:2.0:1787828193.149976:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828193.149977:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:2.0:1787828193.149981:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.149985:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828193.149987:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828193.149988:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828193.149990:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327219840, offset 0
00000400:00000200:2.0:1787828193.149993:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828193.149995:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828193.149998:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828193.150000:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828193.150002:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828193.150018:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828193.150020:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828193.150023:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39489:39489:256:4294967295] 192.168.203.116@tcp LPNI seq info [39489:39489:8:4294967295]
00000400:00000200:2.0:1787828193.150026:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828193.150030:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828193.150034:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828193.150036:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828193.150039:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828193.150041:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787828193.150044:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828193.150046:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787828193.150048:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828193.150049:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828193.150052:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828193.150054:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828193.150056:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828193.150057:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828193.150059:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828193.150062:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150067:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828193.150069:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828218 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150073:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828193.150075:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828193.150077:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828193.150079:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828193.150082:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828193.150083:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297013266
00000100:00000001:2.0:1787828193.150085:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297013932
00000100:00000001:2.0:1787828193.150086:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297013266
00000100:00000001:2.0:1787828193.150088:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297013932
00000100:00000001:2.0:1787828193.150089:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828193.150090:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828193.150091:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828193.150094:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828193/real 1787828193]  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828193.150099:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828193.150100:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828193.150101:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828193.150102:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828193.150103:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828193.150104:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828193.150106:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828193.150111:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150115:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828193.150116:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150118:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828193.150119:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828193.150121:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150125:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828193.150127:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150132:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828193.150134:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150135:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828193.150136:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150137:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828193.150138:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828193.150139:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828193.150141:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828193.150142:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828193.150145:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828193.150149:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327219840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828193.150152:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828193.150153:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150157:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828193.150158:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828193.150159:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828193.150161:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828193.150162:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828193.150164:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327219968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828193.150165:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828193.150166:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828193.150168:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828193.150169:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828193.150171:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002000.
02000000:00000001:2.0:1787828193.150172:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828193.150174:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828193.150176:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828193.150178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828193.150180:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327219968, portal 4
00000100:00000001:2.0:1787828193.150182:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828193.150183:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:2.0:1787828193.150186:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150191:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828193.150192:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828193.150194:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828193.150195:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327219968, offset 0
00000400:00000200:2.0:1787828193.150197:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828193.150200:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828193.150202:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828193.150203:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828193.150205:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828193.150207:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828193.150208:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828193.150210:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39490:39490:256:4294967295] 192.168.203.116@tcp LPNI seq info [39490:39490:8:4294967295]
00000400:00000200:2.0:1787828193.150213:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828193.150217:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828193.150219:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828193.150221:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828193.150223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828193.150225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787828193.150228:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828193.150229:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787828193.150232:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828193.150233:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828193.150235:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828193.150237:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828193.150238:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828193.150239:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828193.150241:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828193.150243:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150248:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828193.150250:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828218 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150254:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828193.150256:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828193.150257:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828193.150259:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828193.150261:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828193.150262:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297013271
00000100:00000001:2.0:1787828193.150263:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297013932
00000100:00000001:2.0:1787828193.150265:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297013271
00000100:00000001:2.0:1787828193.150266:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297013932
00000100:00000001:2.0:1787828193.150268:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828193.150269:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828193.150270:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828193.150272:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828193/real 1787828193]  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828193.150277:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828193.150277:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828193.150278:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828193.150279:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828193.150280:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828193.150281:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828193.150283:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828193.150288:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150296:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828193.150297:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150299:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828193.150300:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828193.150302:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150306:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828193.150308:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150312:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828193.150313:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150314:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828193.150315:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150316:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828193.150317:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828193.150319:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828193.150320:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828193.150322:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828193.150324:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828193.150328:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327219968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828193.150330:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150332:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828193.150334:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327219712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150338:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828193.150339:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828193.150341:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787828193.150342:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828193.150344:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828193.150345:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828193.150346:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828193.150347:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:2.0:1787828193.150349:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828193.150350:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828193.150350:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:2.0:1787828193.150352:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828193.150353:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150354:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828193.150356:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327219840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150361:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828193.150361:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828193.150362:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828193.150363:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828193.150364:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828193.150365:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828193.150366:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828193.150367:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:2.0:1787828193.150368:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828193.150369:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828193.150369:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:2.0:1787828193.150370:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828193.150371:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150372:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828193.150374:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327219968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828193.150378:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828193.150378:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828193.150379:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002000.
02000000:00000001:2.0:1787828193.150380:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828193.150381:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828193.150381:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828193.150383:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828193.150384:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:2.0:1787828193.150385:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828193.150386:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828193.150387:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:2.0:1787828193.150388:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828193.150389:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150390:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828193.150392:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828193.150393:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828193.150394:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828193.150395:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828193.852736:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297013932, at 4297013952
00000100:00000010:2.0:1787828193.852753:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828193.980840:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828193.980844:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828193.980847:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828196.028788:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828196.028793:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828196.028797:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828197.052808:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828197.052811:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828197.052814:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828198.269233:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828198.269239:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828198.269727:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828198.269729:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828198.269956:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828198.269958:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828198.270161:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828198.270174:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828198.972738:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297019052, at 4297019072
00000100:00000010:2.0:1787828198.972764:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f960.
00000020:00000001:2.0:1787828199.100841:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828199.100845:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828199.100848:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828200.124869:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828200.124874:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828200.124878:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828202.172870:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828202.172873:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828202.172876:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828203.196840:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828203.196843:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828203.196847:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828203.388971:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828203.388977:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828203.388981:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828203.388991:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389012:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828203.389015:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.389021:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828203.389025:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828203.389029:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828203.389033:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327220608:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828203.389040:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828203.389045:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828203.389049:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.389056:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828203.389063:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828203.389067:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828203.389076:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828203.389082:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828203.389089:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828203.389097:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327220608, portal 25
00000100:00000001:2.0:1787828203.389103:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828203.389106:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:2.0:1787828203.389118:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389147:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828203.389154:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828203.389158:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828203.389163:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327220608, offset 0
00000400:00000200:2.0:1787828203.389173:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828203.389185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.389195:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828203.389205:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828203.389211:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828203.389216:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828203.389222:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828203.389231:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39495:39495:256:4294967295] 192.168.203.116@tcp LPNI seq info [39495:39495:8:4294967295]
00000400:00000200:2.0:1787828203.389240:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.389252:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828203.389262:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.389267:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828203.389276:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828203.389282:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828203.389290:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828203.389294:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787828203.389303:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.389308:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828203.389315:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828203.389323:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828203.389338:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828203.389342:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828203.389350:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828203.389358:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389379:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.389385:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389403:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389408:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828203.389414:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828203.389424:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828203.389428:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297024172
00000100:00000010:2.0:1787828203.389435:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828203.389443:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828203.389450:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297024172
00000100:00000010:2.0:1787828203.389458:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9f00.
00000100:00000001:2.0:1787828203.389464:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828203.389468:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828203.389471:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389474:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828203.389481:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828203/real 1787828203]  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828203.389498:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828203.389501:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828203.389504:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828203.389509:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828203.389511:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828203.389514:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828203.389520:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828203.389526:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389531:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828203.389532:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389534:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.389535:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828203.389537:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389542:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828203.389545:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389549:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828203.389551:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389552:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.389554:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389556:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828203.389557:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828203.389559:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828203.389562:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828203.389564:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828203.389566:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828203.389572:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327220608:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828203.389575:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389576:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.389578:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327220608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389582:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828203.389584:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828203.389586:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828203.389588:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828203.389589:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828203.389591:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828203.389592:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828203.389594:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:2.0:1787828203.389597:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828203.389598:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828203.389599:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:2.0:1787828203.389601:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828203.389602:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389604:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389608:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828203.389610:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389611:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828203.389612:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828203.389613:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828203.389616:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389621:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828203.389622:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.389623:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828203.389624:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828203.389626:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828203.389628:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327220736:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828203.389629:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828203.389631:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828203.389632:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.389633:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828203.389635:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828203.389636:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828203.389639:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828203.389641:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828203.389643:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828203.389645:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327220736, portal 10
00000100:00000001:2.0:1787828203.389646:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828203.389647:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:2.0:1787828203.389651:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389683:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828203.389685:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828203.389687:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828203.389689:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327220736, offset 0
00000400:00000200:2.0:1787828203.389692:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828203.389695:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.389698:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828203.389700:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828203.389702:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828203.389703:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828203.389705:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828203.389707:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39496:39496:256:4294967295] 192.168.203.116@tcp LPNI seq info [39496:39496:8:4294967295]
00000400:00000200:2.0:1787828203.389710:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.389715:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828203.389718:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.389720:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828203.389723:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828203.389725:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828203.389728:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828203.389729:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787828203.389732:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.389733:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828203.389735:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828203.389738:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828203.389740:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828203.389741:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828203.389743:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828203.389745:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389751:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.389752:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389765:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389766:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828203.389768:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828203.389770:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828203.389773:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828203.389774:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297023499
00000100:00000001:2.0:1787828203.389776:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297024172
00000100:00000001:2.0:1787828203.389777:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297023499
00000100:00000001:2.0:1787828203.389779:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297024172
00000100:00000001:2.0:1787828203.389780:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828203.389782:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389783:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828203.389786:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828203/real 1787828203]  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828203.389792:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828203.389793:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828203.389794:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828203.389795:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828203.389796:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828203.389797:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828203.389799:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828203.389805:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389809:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828203.389810:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389811:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.389812:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828203.389814:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389818:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828203.389821:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389825:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828203.389826:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389827:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.389828:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.389829:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828203.389830:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828203.389832:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828203.389834:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828203.389836:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828203.389838:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828203.389843:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327220736:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828203.389845:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828203.389847:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389851:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828203.389852:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.389853:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828203.389855:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828203.389856:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828203.389858:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327220864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828203.389859:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828203.389860:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828203.389861:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.389863:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828203.389865:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787828203.389866:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828203.389868:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828203.389869:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828203.389871:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828203.389873:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327220864, portal 4
00000100:00000001:2.0:1787828203.389875:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828203.389876:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:2.0:1787828203.389879:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389883:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828203.389885:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828203.389886:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828203.389888:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327220864, offset 0
00000400:00000200:2.0:1787828203.389890:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828203.389892:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.389894:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828203.389896:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828203.389898:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828203.389899:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828203.389901:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828203.389903:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39497:39497:256:4294967295] 192.168.203.116@tcp LPNI seq info [39497:39497:8:4294967295]
00000400:00000200:2.0:1787828203.389906:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.389910:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828203.389912:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.389914:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828203.389917:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828203.389918:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828203.389921:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828203.389922:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787828203.389951:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.389953:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828203.389955:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828203.389957:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828203.389960:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828203.389961:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828203.389963:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828203.389966:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389971:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.389973:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.389977:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389978:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828203.389981:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828203.389983:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828203.389985:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828203.389987:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297023506
00000100:00000001:2.0:1787828203.389989:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297024172
00000100:00000001:2.0:1787828203.389990:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297023506
00000100:00000001:2.0:1787828203.389992:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297024172
00000100:00000001:2.0:1787828203.389993:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828203.389994:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.389995:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828203.389998:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828203/real 1787828203]  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828203.390002:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828203.390003:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828203.390005:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828203.390006:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828203.390007:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828203.390009:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828203.390010:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828203.390016:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390020:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828203.390021:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390022:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.390023:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828203.390025:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390029:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828203.390031:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390036:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828203.390037:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390038:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.390039:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390040:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828203.390041:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828203.390043:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828203.390045:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828203.390046:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828203.390048:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828203.390053:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327220864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828203.390055:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828203.390057:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390061:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828203.390062:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.390063:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828203.390064:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828203.390065:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828203.390067:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327220992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828203.390069:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828203.390070:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828203.390071:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828203.390072:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828203.390075:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002000.
02000000:00000001:2.0:1787828203.390077:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828203.390079:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828203.390081:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828203.390083:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828203.390084:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327220992, portal 4
00000100:00000001:2.0:1787828203.390086:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828203.390087:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:2.0:1787828203.390090:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390095:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828203.390096:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828203.390097:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828203.390099:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327220992, offset 0
00000400:00000200:2.0:1787828203.390101:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828203.390103:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.390105:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828203.390107:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828203.390109:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828203.390111:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828203.390112:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828203.390115:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39498:39498:256:4294967295] 192.168.203.116@tcp LPNI seq info [39498:39498:8:4294967295]
00000400:00000200:2.0:1787828203.390117:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828203.390121:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828203.390124:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.390126:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828203.390128:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828203.390130:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787828203.390133:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828203.390135:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787828203.390137:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828203.390138:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828203.390140:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828203.390142:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828203.390144:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828203.390145:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828203.390146:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828203.390149:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390153:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.390155:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390159:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.390161:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828203.390162:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828203.390164:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828203.390166:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828203.390167:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297023511
00000100:00000001:2.0:1787828203.390169:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297024172
00000100:00000001:2.0:1787828203.390170:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297023511
00000100:00000001:2.0:1787828203.390171:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297024172
00000100:00000001:2.0:1787828203.390173:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828203.390174:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.390175:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828203.390177:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828203/real 1787828203]  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828203.390181:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828203.390182:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828203.390183:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828203.390184:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828203.390185:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828203.390186:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828203.390188:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828203.390193:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390201:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828203.390202:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390203:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.390204:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828203.390206:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390210:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828203.390212:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390216:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828203.390218:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390219:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828203.390219:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390221:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828203.390222:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828203.390223:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828203.390225:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828203.390226:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828203.390228:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828203.390233:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327220992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828203.390236:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390237:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.390239:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327220736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390243:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828203.390244:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828203.390246:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828203.390247:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828203.390248:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828203.390249:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828203.390251:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828203.390252:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:2.0:1787828203.390253:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828203.390254:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828203.390255:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:2.0:1787828203.390258:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828203.390259:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390260:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.390262:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327220864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390266:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828203.390267:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828203.390268:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787828203.390270:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828203.390270:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828203.390271:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828203.390272:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828203.390273:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:2.0:1787828203.390274:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828203.390275:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828203.390276:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787828203.390277:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828203.390278:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390279:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828203.390280:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327220992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828203.390284:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828203.390285:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828203.390286:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002000.
02000000:00000001:2.0:1787828203.390287:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828203.390287:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828203.390288:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828203.390289:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828203.390290:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:2.0:1787828203.390292:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828203.390292:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828203.390293:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:2.0:1787828203.390295:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828203.390295:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390296:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828203.390299:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828203.390300:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828203.390301:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828203.390302:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828204.092753:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297024172, at 4297024192
00000100:00000010:2.0:1787828204.092767:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828205.244853:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828205.244857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828205.244860:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828206.268831:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828206.268836:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828206.268840:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828208.316916:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828208.316919:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828208.316921:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828208.509311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828208.509317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828208.509784:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828208.509786:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828208.509983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828208.509984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828208.510158:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828208.510160:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828209.212735:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297029292, at 4297029312
00000100:00000010:2.0:1787828209.212754:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f120.
00000020:00000001:2.0:1787828209.340804:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828209.340807:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828209.340810:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828211.388873:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828211.388891:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828211.388895:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828212.412852:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828212.412856:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828212.412859:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828213.629055:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828213.629064:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828213.629069:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828213.629081:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629100:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828213.629103:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.629109:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828213.629115:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828213.629122:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828213.629131:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327221632:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828213.629139:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828213.629146:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828213.629152:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.629158:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828213.629166:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828213.629171:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828213.629180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828213.629186:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828213.629192:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828213.629200:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327221632, portal 25
00000100:00000001:2.0:1787828213.629204:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828213.629206:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000100:00000040:2.0:1787828213.629214:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629223:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828213.629227:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828213.629232:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828213.629235:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327221632, offset 0
00000400:00000200:2.0:1787828213.629242:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828213.629253:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.629261:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828213.629266:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828213.629271:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828213.629276:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828213.629279:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828213.629285:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39503:39503:256:4294967295] 192.168.203.116@tcp LPNI seq info [39503:39503:8:4294967295]
00000400:00000200:2.0:1787828213.629295:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.629306:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828213.629315:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.629320:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828213.629329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828213.629335:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787828213.629344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828213.629349:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787828213.629354:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.629358:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828213.629363:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828213.629368:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828213.629374:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828213.629378:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828213.629382:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828213.629389:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629403:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.629410:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629422:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.629425:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828213.629431:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828213.629437:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828213.629442:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297034412
00000100:00000010:2.0:1787828213.629449:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828213.629458:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828213.629465:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297034412
00000100:00000010:2.0:1787828213.629470:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9540.
00000100:00000001:2.0:1787828213.629476:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828213.629481:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828213.629485:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.629488:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828213.629495:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828213/real 1787828213]  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828213.629509:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828213.629512:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828213.629515:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828213.629519:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828213.629521:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828213.629527:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828213.629533:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828213.629547:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629556:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828213.629560:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.629563:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.629566:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828213.629570:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629582:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828213.629587:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629594:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828213.629607:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.629612:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.629614:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.629618:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828213.629621:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828213.629625:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828213.629631:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828213.629636:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828213.629643:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828213.629689:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327221632:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828213.629693:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.629697:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.629702:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327221632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629711:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828213.629715:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828213.629718:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828213.629723:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828213.629728:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828213.629731:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828213.629735:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828213.629740:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:2.0:1787828213.629745:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828213.629748:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828213.629750:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:2.0:1787828213.629754:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828213.629757:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.629759:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.629765:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828213.629768:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.629772:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828213.629775:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828213.629777:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828213.629783:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629797:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828213.629801:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.629807:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828213.629811:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828213.629817:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828213.629828:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327221760:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828213.629837:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828213.629842:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828213.629846:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.629851:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828213.629858:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828213.629876:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828213.629885:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828213.629890:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828213.629897:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828213.629904:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327221760, portal 10
00000100:00000001:2.0:1787828213.629908:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828213.629911:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00000040:2.0:1787828213.629919:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.629928:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828213.629932:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828213.629938:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828213.629943:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327221760, offset 0
00000400:00000200:2.0:1787828213.629949:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828213.629954:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.629958:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828213.629963:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828213.629968:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828213.629973:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828213.629975:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828213.629982:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39504:39504:256:4294967295] 192.168.203.116@tcp LPNI seq info [39504:39504:8:4294967295]
00000400:00000200:2.0:1787828213.629993:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.630001:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828213.630010:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.630015:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828213.630023:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828213.630028:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787828213.630036:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828213.630038:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787828213.630046:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.630048:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828213.630054:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828213.630057:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828213.630059:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828213.630060:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828213.630062:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828213.630065:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630071:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.630073:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630078:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630079:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828213.630082:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828213.630084:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828213.630086:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828213.630088:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297033739
00000100:00000001:2.0:1787828213.630090:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297034412
00000100:00000001:2.0:1787828213.630091:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297033739
00000100:00000001:2.0:1787828213.630093:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297034412
00000100:00000001:2.0:1787828213.630094:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828213.630095:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630097:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828213.630099:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828213/real 1787828213]  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828213.630105:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828213.630106:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828213.630107:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828213.630108:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828213.630109:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828213.630111:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828213.630112:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828213.630117:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630122:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828213.630123:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630124:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.630125:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828213.630127:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630131:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828213.630134:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630138:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828213.630139:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630140:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.630141:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630142:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828213.630143:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828213.630145:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828213.630147:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828213.630148:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828213.630151:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828213.630156:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327221760:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828213.630159:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828213.630161:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630165:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828213.630165:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.630167:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828213.630168:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828213.630169:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828213.630171:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327221888:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828213.630173:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828213.630174:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828213.630175:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.630176:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828213.630178:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828213.630180:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828213.630182:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828213.630183:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828213.630186:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828213.630188:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327221888, portal 4
00000100:00000001:2.0:1787828213.630189:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828213.630190:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:2.0:1787828213.630194:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630198:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828213.630200:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828213.630201:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828213.630203:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327221888, offset 0
00000400:00000200:2.0:1787828213.630205:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828213.630208:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.630210:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828213.630212:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828213.630213:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828213.630215:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828213.630217:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828213.630219:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39505:39505:256:4294967295] 192.168.203.116@tcp LPNI seq info [39505:39505:8:4294967295]
00000400:00000200:2.0:1787828213.630222:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.630226:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828213.630229:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.630230:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828213.630233:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828213.630234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787828213.630237:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828213.630238:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787828213.630240:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.630242:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828213.630244:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828213.630246:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828213.630248:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828213.630249:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828213.630251:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828213.630253:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630258:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.630260:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630264:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630265:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828213.630267:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828213.630268:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828213.630270:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828213.630272:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297033746
00000100:00000001:2.0:1787828213.630273:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297034412
00000100:00000001:2.0:1787828213.630275:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297033746
00000100:00000001:2.0:1787828213.630276:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297034412
00000100:00000001:2.0:1787828213.630277:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828213.630279:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630280:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828213.630282:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828213/real 1787828213]  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828213.630286:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828213.630287:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828213.630288:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828213.630289:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828213.630290:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828213.630291:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828213.630293:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828213.630299:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630302:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828213.630303:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630304:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.630305:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828213.630307:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630311:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828213.630313:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630317:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828213.630319:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630320:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.630321:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630322:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828213.630323:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828213.630325:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828213.630327:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828213.630328:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828213.630330:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828213.630335:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327221888:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828213.630337:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630338:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.630340:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327221760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630344:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828213.630345:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828213.630346:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828213.630347:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828213.630348:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828213.630349:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828213.630350:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828213.630351:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:2.0:1787828213.630353:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828213.630354:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828213.630355:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:2.0:1787828213.630356:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828213.630357:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630359:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.630360:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327221888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630369:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828213.630369:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828213.630370:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828213.630372:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828213.630373:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828213.630373:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828213.630374:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828213.630375:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:2.0:1787828213.630376:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828213.630377:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828213.630378:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:2.0:1787828213.630380:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828213.630381:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630382:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630384:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828213.630385:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630386:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828213.630387:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828213.630388:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828213.630390:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630394:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828213.630395:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.630396:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828213.630397:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828213.630398:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828213.630400:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222016:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828213.630402:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828213.630403:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828213.630404:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828213.630405:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828213.630407:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828213.630409:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828213.630411:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828213.630412:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828213.630414:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828213.630416:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222016, portal 4
00000100:00000001:2.0:1787828213.630417:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828213.630418:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:2.0:1787828213.630422:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630425:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828213.630427:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828213.630428:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828213.630430:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327222016, offset 0
00000400:00000200:2.0:1787828213.630432:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828213.630434:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.630436:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828213.630438:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828213.630440:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828213.630441:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828213.630443:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828213.630445:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39506:39506:256:4294967295] 192.168.203.116@tcp LPNI seq info [39506:39506:8:4294967295]
00000400:00000200:2.0:1787828213.630448:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828213.630452:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828213.630454:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.630456:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828213.630458:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828213.630460:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787828213.630463:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828213.630465:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787828213.630468:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828213.630469:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828213.630471:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828213.630473:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828213.630474:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828213.630475:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828213.630477:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828213.630480:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630485:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.630486:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630491:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630492:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828213.630493:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828213.630495:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828213.630497:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828213.630498:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297033751
00000100:00000001:2.0:1787828213.630499:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297034412
00000100:00000001:2.0:1787828213.630501:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297033751
00000100:00000001:2.0:1787828213.630502:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297034412
00000100:00000001:2.0:1787828213.630504:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828213.630505:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630506:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828213.630508:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828213/real 1787828213]  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828213.630512:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828213.630513:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828213.630514:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828213.630515:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828213.630516:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828213.630517:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828213.630519:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828213.630524:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630532:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828213.630534:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630535:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.630536:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828213.630538:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630542:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828213.630544:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630548:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828213.630549:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630550:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828213.630551:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630552:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828213.630553:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828213.630555:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828213.630557:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828213.630558:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828213.630560:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828213.630565:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222016:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828213.630567:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630568:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828213.630570:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327222016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828213.630574:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828213.630575:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828213.630577:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828213.630578:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828213.630579:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828213.630580:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828213.630581:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828213.630582:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:2.0:1787828213.630583:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828213.630584:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828213.630585:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787828213.630586:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828213.630587:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630588:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828213.630590:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828213.630591:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828213.630592:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828213.630593:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828214.332761:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297034412, at 4297034432
00000100:00000010:2.0:1787828214.332779:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828214.460915:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828214.460923:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828214.460929:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828215.484846:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828215.484863:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828215.484867:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828217.532813:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828217.532817:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828217.532820:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828218.556802:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828218.556806:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828218.556808:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828218.748938:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828218.748943:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828218.748948:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828218.748957:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.748978:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828218.748981:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.748986:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828218.748991:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828218.748994:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828218.749002:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327222144:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828218.749008:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828218.749013:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828218.749016:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.749020:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828218.749027:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828218.749031:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828218.749039:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828218.749046:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828218.749055:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828218.749066:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222144, portal 25
00000100:00000001:2.0:1787828218.749072:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828218.749077:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:2.0:1787828218.749090:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749114:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828218.749119:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828218.749125:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828218.749132:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327222144, offset 0
00000400:00000200:2.0:1787828218.749143:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828218.749155:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.749164:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828218.749172:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828218.749181:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828218.749185:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828218.749188:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828218.749194:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39507:39507:256:4294967295] 192.168.203.116@tcp LPNI seq info [39507:39507:8:4294967295]
00000400:00000200:2.0:1787828218.749201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.749215:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828218.749224:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.749233:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828218.749241:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828218.749248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828218.749257:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828218.749261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787828218.749268:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.749272:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828218.749278:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828218.749287:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828218.749295:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828218.749299:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828218.749304:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828218.749313:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749332:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.749338:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749358:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749362:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828218.749368:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828218.749375:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828218.749380:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297039532
00000100:00000010:2.0:1787828218.749387:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828218.749396:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828218.749403:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297039532
00000100:00000010:2.0:1787828218.749410:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9b40.
00000100:00000001:2.0:1787828218.749416:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828218.749419:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828218.749425:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749430:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828218.749439:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828218/real 1787828218]  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828218.749465:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828218.749468:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828218.749472:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828218.749477:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828218.749480:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828218.749485:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828218.749491:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828218.749499:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749504:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828218.749506:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749507:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.749509:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828218.749512:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749516:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828218.749519:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749524:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828218.749525:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749527:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.749527:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749529:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828218.749530:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828218.749532:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828218.749535:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828218.749536:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828218.749539:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828218.749545:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327222144:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828218.749548:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749550:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.749552:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327222144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749556:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828218.749558:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828218.749560:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828218.749561:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828218.749563:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828218.749564:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828218.749565:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828218.749567:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:2.0:1787828218.749570:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828218.749571:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828218.749572:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:2.0:1787828218.749575:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828218.749576:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749578:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749582:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828218.749583:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749585:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828218.749586:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828218.749587:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828218.749589:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749594:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828218.749595:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.749597:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828218.749598:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828218.749599:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828218.749601:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222272:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828218.749603:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828218.749605:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828218.749606:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.749607:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828218.749609:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828218.749610:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828218.749613:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828218.749615:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828218.749617:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828218.749619:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222272, portal 10
00000100:00000001:2.0:1787828218.749620:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828218.749621:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:2.0:1787828218.749626:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749630:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828218.749632:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828218.749633:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828218.749635:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327222272, offset 0
00000400:00000200:2.0:1787828218.749637:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828218.749639:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.749641:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828218.749643:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828218.749645:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828218.749647:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828218.749648:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828218.749650:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39508:39508:256:4294967295] 192.168.203.116@tcp LPNI seq info [39508:39508:8:4294967295]
00000400:00000200:2.0:1787828218.749653:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.749675:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828218.749678:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.749681:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828218.749684:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828218.749686:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828218.749689:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828218.749691:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787828218.749694:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.749695:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828218.749697:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828218.749700:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828218.749702:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828218.749703:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828218.749705:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828218.749707:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749712:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.749714:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749718:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749720:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828218.749721:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828218.749724:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828218.749726:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828218.749728:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297038859
00000100:00000001:2.0:1787828218.749730:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297039532
00000100:00000001:2.0:1787828218.749731:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297038859
00000100:00000001:2.0:1787828218.749733:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297039532
00000100:00000001:2.0:1787828218.749734:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828218.749735:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749736:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828218.749739:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828218/real 1787828218]  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828218.749743:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828218.749744:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828218.749746:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828218.749747:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828218.749748:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828218.749750:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828218.749752:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828218.749757:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749761:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828218.749761:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749763:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.749763:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828218.749765:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749769:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828218.749772:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749776:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828218.749778:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749779:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.749780:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749781:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828218.749782:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828218.749784:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828218.749786:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828218.749787:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828218.749789:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828218.749794:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222272:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828218.749797:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828218.749799:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749803:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828218.749804:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.749806:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828218.749807:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828218.749808:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828218.749809:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222400:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828218.749811:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828218.749812:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828218.749813:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.749815:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828218.749817:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828218.749818:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828218.749821:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828218.749822:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828218.749824:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828218.749839:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222400, portal 4
00000100:00000001:2.0:1787828218.749841:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828218.749842:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:2.0:1787828218.749847:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749851:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828218.749853:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828218.749854:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828218.749856:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327222400, offset 0
00000400:00000200:2.0:1787828218.749859:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828218.749862:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.749864:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828218.749866:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828218.749868:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828218.749870:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828218.749872:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828218.749874:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39509:39509:256:4294967295] 192.168.203.116@tcp LPNI seq info [39509:39509:8:4294967295]
00000400:00000200:2.0:1787828218.749876:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.749880:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828218.749883:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.749885:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828218.749887:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828218.749889:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828218.749892:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828218.749894:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787828218.749896:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.749898:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828218.749900:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828218.749902:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828218.749904:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828218.749905:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828218.749907:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828218.749909:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749914:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.749916:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749920:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749921:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828218.749923:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828218.749925:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828218.749927:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828218.749929:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297038866
00000100:00000001:2.0:1787828218.749931:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297039532
00000100:00000001:2.0:1787828218.749932:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297038866
00000100:00000001:2.0:1787828218.749934:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297039532
00000100:00000001:2.0:1787828218.749935:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828218.749937:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.749938:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828218.749941:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828218/real 1787828218]  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828218.749945:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828218.749946:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828218.749947:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828218.749949:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828218.749950:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828218.749951:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828218.749953:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828218.749958:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749963:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828218.749963:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749964:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.749965:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828218.749967:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749971:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828218.749974:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.749978:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828218.749979:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749980:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.749981:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.749982:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828218.749983:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828218.749985:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828218.749987:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828218.749989:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828218.749991:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828218.749996:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222400:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828218.749998:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828218.750000:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750003:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828218.750004:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.750006:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828218.750007:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828218.750008:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828218.750010:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222528:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828218.750012:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828218.750013:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828218.750014:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828218.750015:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828218.750017:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787828218.750019:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828218.750020:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828218.750022:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828218.750024:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828218.750026:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222528, portal 4
00000100:00000001:2.0:1787828218.750027:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828218.750028:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:2.0:1787828218.750031:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750036:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828218.750038:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828218.750039:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828218.750041:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327222528, offset 0
00000400:00000200:2.0:1787828218.750043:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828218.750046:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.750049:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828218.750051:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828218.750054:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828218.750056:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828218.750058:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828218.750061:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39510:39510:256:4294967295] 192.168.203.116@tcp LPNI seq info [39510:39510:8:4294967295]
00000400:00000200:2.0:1787828218.750065:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828218.750071:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828218.750077:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.750080:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828218.750084:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828218.750087:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787828218.750091:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828218.750094:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787828218.750098:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828218.750100:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828218.750103:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828218.750105:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828218.750107:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce65d8
00000400:00000010:2.0:1787828218.750109:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce65d8.
00000100:00000001:2.0:1787828218.750110:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828218.750113:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750118:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.750120:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750128:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.750129:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828218.750132:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828218.750134:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828218.750136:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828218.750138:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297038871
00000100:00000001:2.0:1787828218.750139:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297039532
00000100:00000001:2.0:1787828218.750141:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297038871
00000100:00000001:2.0:1787828218.750142:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297039532
00000100:00000001:2.0:1787828218.750143:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828218.750145:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.750146:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828218.750148:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828218/real 1787828218]  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828218.750153:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828218.750154:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828218.750155:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828218.750157:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828218.750158:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828218.750159:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828218.750161:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828218.750166:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750170:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828218.750171:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750172:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.750173:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828218.750175:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750180:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828218.750183:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750187:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828218.750188:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750189:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828218.750190:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750191:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828218.750192:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828218.750194:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828218.750196:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828218.750197:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828218.750199:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828218.750204:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222528:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828218.750206:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750207:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.750209:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327222272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750213:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828218.750214:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828218.750215:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828218.750217:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828218.750218:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828218.750219:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828218.750220:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828218.750222:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:2.0:1787828218.750224:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828218.750225:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828218.750226:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:2.0:1787828218.750227:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828218.750228:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750229:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.750231:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327222400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750236:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828218.750236:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828218.750237:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828218.750238:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828218.750239:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828218.750240:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828218.750241:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828218.750242:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:2.0:1787828218.750243:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828218.750244:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828218.750244:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:2.0:1787828218.750246:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828218.750246:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750247:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828218.750249:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327222528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828218.750253:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828218.750253:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828218.750254:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787828218.750256:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828218.750256:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828218.750257:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828218.750258:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828218.750259:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:2.0:1787828218.750260:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828218.750260:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828218.750261:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:2.0:1787828218.750263:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828218.750264:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750265:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828218.750267:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828218.750268:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828218.750269:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828218.750271:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828219.452751:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297039532, at 4297039552
00000100:00000010:2.0:1787828219.452768:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828220.604901:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828220.604906:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828220.604910:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828221.628817:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828221.628821:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828221.628825:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828223.676840:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828223.676844:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828223.676847:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828223.868971:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828223.868977:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828223.868982:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828223.868994:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869015:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828223.869021:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869028:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828223.869033:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828223.869042:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828223.869050:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327222656:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828223.869056:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828223.869062:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828223.869065:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869071:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828223.869079:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828223.869084:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828223.869092:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828223.869099:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828223.869108:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828223.869117:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222656, portal 25
00000100:00000001:2.0:1787828223.869124:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828223.869128:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:2.0:1787828223.869140:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869157:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828223.869164:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828223.869168:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce65d8.
00000100:00000200:2.0:1787828223.869173:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327222656, offset 0
00000400:00000200:2.0:1787828223.869193:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869214:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828223.869222:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828223.869229:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828223.869235:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828223.869240:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869246:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39511:39511:256:4294967295] 192.168.203.116@tcp LPNI seq info [39511:39511:8:4294967295]
00000400:00000200:2.0:1787828223.869258:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869275:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828223.869290:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828223.869298:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828223.869310:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828223.869324:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828223.869357:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828223.869368:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828223.869372:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297044652
00000100:00000010:2.0:1787828223.869380:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828223.869388:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828223.869395:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297044652
00000100:00000010:2.0:1787828223.869401:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9c60.
00000100:00000001:2.0:1787828223.869407:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828223.869418:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828223.869423:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869430:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869436:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828223.869443:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828223.869447:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787828223.869451:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828223.869457:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828223.869461:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828223.869469:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869488:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828223.869490:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869494:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828223.869496:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828223.869502:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828223.869514:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222784:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828223.869518:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828223.869522:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828223.869524:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869525:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828223.869529:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828223.869531:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828223.869535:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828223.869538:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f79f8.
00000400:00000010:2.0:1787828223.869541:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6908.
00000100:00000200:2.0:1787828223.869543:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222784, portal 10
00000100:00000001:2.0:1787828223.869545:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828223.869546:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:2.0:1787828223.869551:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869557:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828223.869559:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828223.869560:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6770.
00000100:00000200:2.0:1787828223.869562:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327222784, offset 0
00000400:00000200:2.0:1787828223.869565:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869568:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869571:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828223.869574:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828223.869576:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828223.869577:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828223.869579:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869582:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39512:39512:255:4294967295] 192.168.203.116@tcp LPNI seq info [39512:39512:7:4294967295]
00000400:00000200:2.0:1787828223.869585:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869589:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828223.869593:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828223.869596:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37b00.
00000800:00000200:2.0:1787828223.869599:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828223.869602:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828223.869604:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828223.869607:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828223.869609:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828223.869610:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297043978
00000100:00000001:2.0:1787828223.869612:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297044652
00000100:00000001:2.0:1787828223.869614:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297043978
00000100:00000001:2.0:1787828223.869616:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297044652
00000100:00000001:2.0:1787828223.869618:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828223.869619:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869620:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828223.869623:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869628:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828223.869628:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869630:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828223.869631:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828223.869633:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828223.869635:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222912:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828223.869637:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828223.869639:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828223.869640:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869641:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828223.869643:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828223.869644:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828223.869646:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828223.869648:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828223.869650:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828223.869652:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327222912, portal 4
00000100:00000001:2.0:1787828223.869653:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828223.869680:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:2.0:1787828223.869685:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869690:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828223.869692:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828223.869694:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6330.
00000100:00000200:2.0:1787828223.869696:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327222912, offset 0
00000400:00000200:2.0:1787828223.869699:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869701:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869704:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828223.869706:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828223.869708:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828223.869709:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828223.869711:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869713:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39513:39513:254:4294967295] 192.168.203.116@tcp LPNI seq info [39513:39513:6:4294967295]
00000400:00000200:2.0:1787828223.869716:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869720:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828223.869724:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828223.869726:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37700.
00000800:00000200:2.0:1787828223.869729:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828223.869732:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828223.869734:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828223.869736:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828223.869738:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828223.869739:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297043986
00000100:00000001:2.0:1787828223.869741:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297044652
00000100:00000001:2.0:1787828223.869743:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297043986
00000100:00000001:2.0:1787828223.869744:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297044652
00000100:00000001:2.0:1787828223.869746:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828223.869747:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869749:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869750:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828223.869752:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828223.869753:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787828223.869754:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828223.869755:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828223.869756:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828223.869759:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869764:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828223.869765:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869766:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828223.869767:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828223.869769:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828223.869771:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223040:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828223.869773:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828223.869775:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828223.869776:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828223.869777:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828223.869779:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001800.
02000000:00000001:2.0:1787828223.869780:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828223.869782:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828223.869784:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7630.
00000400:00000010:2.0:1787828223.869787:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce62a8.
00000100:00000200:2.0:1787828223.869806:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327223040, portal 4
00000100:00000001:2.0:1787828223.869808:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828223.869810:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:2.0:1787828223.869816:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828223.869823:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828223.869826:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828223.869829:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6000.
00000100:00000200:2.0:1787828223.869832:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327223040, offset 0
00000400:00000200:2.0:1787828223.869836:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869840:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869844:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828223.869847:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828223.869850:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828223.869853:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828223.869855:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828223.869858:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39514:39514:253:4294967295] 192.168.203.116@tcp LPNI seq info [39514:39514:5:4294967295]
00000400:00000200:2.0:1787828223.869862:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828223.869868:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828223.869873:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828223.869876:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37f00.
00000800:00000200:2.0:1787828223.869881:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828223.869885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828223.869888:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828223.869892:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828223.869895:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828223.869897:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297043991
00000100:00000001:2.0:1787828223.869900:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297044652
00000100:00000001:2.0:1787828223.869902:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297043991
00000100:00000001:2.0:1787828223.869904:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297044652
00000100:00000001:2.0:1787828223.869906:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828223.869908:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869911:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869913:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828223.869916:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828223.869918:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787828223.869920:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828223.869921:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828223.869923:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828223.869925:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828224.572721:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297044652, at 4297044672
00000100:00000010:2.0:1787828224.572728:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828224.700878:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828224.700881:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828224.700884:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828226.748848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828226.748852:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828226.748856:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828226.941131:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828226.941138:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828226.941145:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828226.941160:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828223/real 1787828226]  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828226.941203:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828226.941207:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828226.941217:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828226.941224:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6e58
00000400:00000010:2.0:1787828226.941226:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6e58.
00000100:00000001:2.0:1787828226.941230:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828226.941237:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828226.941256:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941271:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828226.941274:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941280:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941283:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828226.941293:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941309:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828226.941318:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941333:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828226.941342:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941346:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941347:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941351:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828226.941355:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828226.941362:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828226.941370:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828226.941376:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828226.941387:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828226.941403:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327222656:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828226.941409:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828226.941412:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828226.941415:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327222656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941427:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828226.941430:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828226.941433:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828226.941437:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828226.941440:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828226.941441:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828226.941443:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828226.941446:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:2.0:1787828226.941449:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828226.941451:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828226.941454:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:2.0:1787828226.941458:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828226.941459:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941462:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828226.941463:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828226.941465:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828226.941466:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828226.941470:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828223/real 1787828226]  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828226.941478:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828226.941480:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828226.941482:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7630.
00000400:00000200:2.0:1787828226.941484:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce62a8
00000400:00000010:2.0:1787828226.941485:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce62a8.
00000100:00000001:2.0:1787828226.941488:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828226.941498:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828226.941507:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941514:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828226.941515:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941517:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941518:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828226.941522:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941529:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828226.941533:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941540:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828226.941543:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941544:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941545:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941547:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828226.941549:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828226.941551:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828226.941555:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828226.941557:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828226.941561:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828226.941570:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223040:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828226.941574:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828226.941577:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828223/real 1787828226]  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828226.941585:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828226.941586:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828226.941588:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f79f8.
00000400:00000200:2.0:1787828226.941591:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6908
00000400:00000010:2.0:1787828226.941593:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6908.
00000100:00000001:2.0:1787828226.941595:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828226.941598:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828226.941614:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941621:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828226.941622:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941624:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941625:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828226.941629:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941635:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828226.941640:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941646:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828226.941648:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941649:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941650:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941652:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828226.941682:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828226.941686:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828226.941691:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828226.941694:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828226.941699:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828226.941707:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222784:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828226.941712:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828226.941716:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828223/real 1787828226]  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828226.941730:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828226.941732:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828226.941735:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828226.941738:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828226.941740:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828226.941742:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828226.941746:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828226.941753:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941759:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828226.941760:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941762:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941763:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828226.941766:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941786:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828226.941790:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941796:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828226.941799:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941801:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828226.941802:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941804:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828226.941806:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828226.941809:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828226.941813:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828226.941816:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828226.941821:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828226.941831:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327222912:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828226.941835:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941837:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828226.941840:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327223040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941847:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828226.941848:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828226.941850:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001800.
02000000:00000001:2.0:1787828226.941853:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828226.941855:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828226.941856:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828226.941858:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828226.941860:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:2.0:1787828226.941862:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828226.941863:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828226.941865:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:2.0:1787828226.941867:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828226.941869:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941870:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828226.941874:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327222784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941879:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828226.941880:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828226.941881:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828226.941884:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828226.941885:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828226.941886:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828226.941888:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828226.941890:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:2.0:1787828226.941892:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828226.941894:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828226.941895:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:2.0:1787828226.941897:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828226.941898:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941899:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828226.941902:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327222912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828226.941908:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828226.941909:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828226.941911:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828226.941914:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828226.941915:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828226.941916:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828226.941918:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828226.941920:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:2.0:1787828226.941921:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828226.941923:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828226.941924:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:2.0:1787828226.941926:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828226.941927:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941929:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828226.941938:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828226.941941:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828226.941943:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828226.941944:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828227.772814:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828227.772817:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828227.772820:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828228.989029:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828228.989038:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828228.989042:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828228.989062:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.989099:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828228.989104:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.989113:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828228.989121:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828228.989129:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828228.989144:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327223168:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828228.989157:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828228.989166:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828228.989171:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.989176:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828228.989192:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828228.989204:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828228.989229:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828228.989237:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828228.989251:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828228.989265:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327223168, portal 25
00000100:00000001:2.0:1787828228.989273:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828228.989277:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:2.0:1787828228.989310:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.989339:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828228.989350:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828228.989357:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6908.
00000100:00000200:2.0:1787828228.989365:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327223168, offset 0
00000400:00000200:2.0:1787828228.989381:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828228.989410:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.989426:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828228.989434:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828228.989439:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828228.989443:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828228.989447:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828228.989451:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39515:39515:256:4294967295] 192.168.203.116@tcp LPNI seq info [39515:39515:8:4294967295]
00000400:00000200:2.0:1787828228.989457:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.989468:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828228.989476:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.989484:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37200.
00000800:00000200:2.0:1787828228.989492:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828228.989505:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828228.989526:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828228.989537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828228.989555:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.989562:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37200.
00000400:00000200:2.0:1787828228.989573:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828228.989584:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828228.989593:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6908
00000400:00000010:2.0:1787828228.989603:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6908.
00000100:00000001:2.0:1787828228.989613:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828228.989625:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.989688:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.989695:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.989704:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.989708:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828228.989716:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828228.989722:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828228.989727:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297049773
00000100:00000010:2.0:1787828228.989741:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828228.989777:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828228.989790:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297049773
00000100:00000010:2.0:1787828228.989813:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee97e0.
00000100:00000001:2.0:1787828228.989822:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828228.989835:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828228.989842:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.989848:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828228.989859:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828228/real 1787828228]  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828228.989946:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828228.989954:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828228.989963:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828228.989974:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828228.989980:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828228.989988:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828228.990003:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828228.990039:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990077:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828228.990082:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990088:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.990093:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828228.990108:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990143:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828228.990164:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990182:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828228.990191:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990204:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.990210:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990215:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828228.990220:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828228.990234:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828228.990244:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828228.990248:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828228.990258:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828228.990279:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327223168:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828228.990287:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990290:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.990293:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327223168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990301:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828228.990304:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828228.990309:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828228.990313:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828228.990315:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828228.990316:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828228.990318:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828228.990320:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:2.0:1787828228.990323:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828228.990325:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828228.990327:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:2.0:1787828228.990329:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828228.990330:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990332:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990337:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828228.990339:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.990341:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828228.990342:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828228.990343:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828228.990348:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990353:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828228.990354:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.990356:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828228.990359:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828228.990360:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828228.990362:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223296:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828228.990365:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828228.990367:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828228.990369:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.990371:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828228.990374:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828228.990375:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828228.990378:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828228.990381:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828228.990384:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828228.990387:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327223296, portal 10
00000100:00000001:2.0:1787828228.990388:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828228.990389:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:2.0:1787828228.990394:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990399:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828228.990400:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828228.990401:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6908.
00000100:00000200:2.0:1787828228.990403:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327223296, offset 0
00000400:00000200:2.0:1787828228.990406:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828228.990410:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.990413:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828228.990416:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828228.990419:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828228.990421:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828228.990422:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828228.990425:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39516:39516:256:4294967295] 192.168.203.116@tcp LPNI seq info [39516:39516:8:4294967295]
00000400:00000200:2.0:1787828228.990428:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.990433:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828228.990436:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.990439:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37200.
00000800:00000200:2.0:1787828228.990442:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828228.990444:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828228.990454:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828228.990467:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828228.990470:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.990472:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37200.
00000400:00000200:2.0:1787828228.990474:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828228.990477:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828228.990480:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6908
00000400:00000010:2.0:1787828228.990481:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6908.
00000100:00000001:2.0:1787828228.990484:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828228.990486:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990492:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.990495:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990503:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.990504:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828228.990508:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828228.990513:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828228.990516:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828228.990518:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297049099
00000100:00000001:2.0:1787828228.990521:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297049773
00000100:00000001:2.0:1787828228.990524:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297049099
00000100:00000001:2.0:1787828228.990526:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297049773
00000100:00000001:2.0:1787828228.990529:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828228.990531:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.990533:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828228.990538:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828228/real 1787828228]  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828228.990547:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828228.990548:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828228.990550:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828228.990553:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828228.990555:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828228.990557:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828228.990561:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828228.990566:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990570:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828228.990571:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990573:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.990574:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828228.990576:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990580:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828228.990583:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990587:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828228.990589:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990590:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.990590:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990592:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828228.990593:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828228.990595:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828228.990597:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828228.990598:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828228.990601:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828228.990606:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223296:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828228.990609:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828228.990611:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990616:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828228.990617:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.990619:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828228.990620:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828228.990621:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828228.990623:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223424:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828228.990625:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828228.990627:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828228.990628:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.990629:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828228.990632:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828228.990633:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828228.990636:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828228.990637:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828228.990640:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828228.990642:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327223424, portal 4
00000100:00000001:2.0:1787828228.990643:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828228.990644:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:2.0:1787828228.990648:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990653:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828228.990678:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828228.990680:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6908.
00000100:00000200:2.0:1787828228.990681:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327223424, offset 0
00000400:00000200:2.0:1787828228.990684:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828228.990687:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.990690:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828228.990692:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828228.990694:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828228.990696:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828228.990698:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828228.990700:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39517:39517:256:4294967295] 192.168.203.116@tcp LPNI seq info [39517:39517:8:4294967295]
00000400:00000200:2.0:1787828228.990702:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.990707:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828228.990709:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.990711:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37200.
00000800:00000200:2.0:1787828228.990714:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828228.990716:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828228.990719:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828228.990721:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828228.990723:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.990725:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37200.
00000400:00000200:2.0:1787828228.990727:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828228.990729:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828228.990731:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6908
00000400:00000010:2.0:1787828228.990733:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6908.
00000100:00000001:2.0:1787828228.990735:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828228.990737:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990742:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.990744:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990748:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.990749:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828228.990751:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828228.990768:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828228.990772:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828228.990773:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297049107
00000100:00000001:2.0:1787828228.990775:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297049773
00000100:00000001:2.0:1787828228.990777:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297049107
00000100:00000001:2.0:1787828228.990778:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297049773
00000100:00000001:2.0:1787828228.990780:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828228.990781:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.990783:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828228.990786:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828228/real 1787828228]  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828228.990791:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828228.990792:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828228.990793:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828228.990794:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828228.990796:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828228.990797:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828228.990799:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828228.990804:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990811:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828228.990813:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990814:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.990816:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828228.990820:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990827:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828228.990831:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990838:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828228.990841:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990843:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.990844:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.990847:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828228.990849:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828228.990851:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828228.990855:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828228.990857:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828228.990861:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828228.990870:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223424:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828228.990874:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828228.990878:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990885:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828228.990886:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.990889:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828228.990890:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828228.990892:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828228.990896:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223552:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828228.990899:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828228.990901:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828228.990903:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828228.990905:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828228.990908:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001800.
02000000:00000001:2.0:1787828228.990911:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828228.990914:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828228.990917:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828228.990920:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828228.990923:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327223552, portal 4
00000100:00000001:2.0:1787828228.990925:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828228.990927:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:2.0:1787828228.990933:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.990939:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828228.990941:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828228.990942:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6908.
00000100:00000200:2.0:1787828228.990944:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327223552, offset 0
00000400:00000200:2.0:1787828228.990947:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828228.990949:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.990952:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828228.990954:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828228.990956:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828228.990958:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828228.990959:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828228.990961:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39518:39518:256:4294967295] 192.168.203.116@tcp LPNI seq info [39518:39518:8:4294967295]
00000400:00000200:2.0:1787828228.990964:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828228.990968:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828228.990971:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.990973:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37200.
00000800:00000200:2.0:1787828228.990976:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828228.990978:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828228.990980:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828228.990982:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828228.990985:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828228.990986:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37200.
00000400:00000200:2.0:1787828228.990988:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828228.990991:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828228.990992:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6908
00000400:00000010:2.0:1787828228.990994:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6908.
00000100:00000001:2.0:1787828228.990995:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828228.990998:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991003:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.991005:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991014:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.991015:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828228.991017:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828228.991019:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828228.991021:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828228.991022:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297049112
00000100:00000001:2.0:1787828228.991024:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297049773
00000100:00000001:2.0:1787828228.991025:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297049112
00000100:00000001:2.0:1787828228.991027:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297049773
00000100:00000001:2.0:1787828228.991028:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828228.991030:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.991031:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828228.991033:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828228/real 1787828228]  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828228.991038:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828228.991038:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828228.991039:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828228.991041:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828228.991042:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828228.991043:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828228.991045:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828228.991050:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991054:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828228.991055:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991056:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.991057:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828228.991059:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991064:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828228.991066:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991070:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828228.991072:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991073:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828228.991073:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991074:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828228.991076:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828228.991077:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828228.991079:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828228.991080:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828228.991083:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828228.991087:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327223552:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828228.991089:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991091:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.991093:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327223296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991097:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828228.991099:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828228.991100:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828228.991102:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828228.991103:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828228.991104:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828228.991105:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828228.991107:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:2.0:1787828228.991109:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828228.991110:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828228.991111:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:2.0:1787828228.991113:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828228.991114:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991115:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.991117:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327223424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991121:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828228.991122:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828228.991123:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787828228.991124:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828228.991125:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828228.991126:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828228.991127:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828228.991127:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:2.0:1787828228.991129:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828228.991129:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828228.991130:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:2.0:1787828228.991132:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828228.991133:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991134:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828228.991136:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327223552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828228.991139:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828228.991140:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828228.991141:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001800.
02000000:00000001:2.0:1787828228.991142:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828228.991143:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828228.991143:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828228.991144:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828228.991145:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:2.0:1787828228.991147:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828228.991148:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828228.991149:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:2.0:1787828228.991150:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828228.991151:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991152:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828228.991155:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828228.991156:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828228.991157:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828228.991159:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828229.692797:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297049773, at 4297049792
00000100:00000010:2.0:1787828229.692814:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828229.820859:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828229.820866:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828229.820870:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828230.844892:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828230.844899:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828230.844904:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828232.892851:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828232.892858:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828232.892863:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828233.916959:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828233.916982:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828233.916989:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828234.108797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828234.108805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828234.108806:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828234.108810:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828234.108813:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828234.108819:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2381
00000400:00000200:2.0:1787828234.108826:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828234.108831:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828234.108834:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828234.108836:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.108838:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.108840:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.108842:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828234.108845:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828234.108847:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828234.108849:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828234.108852:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828234.108854:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828234.108856:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828234.108858:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.108859:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.108860:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.108861:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828234.108863:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828234.108866:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828234.108867:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828234.108869:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828234.108872:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828234.108874:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.108876:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828234.108877:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828234.108896:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828234.108899:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828234.108902:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828234.108903:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.108906:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828234.108908:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828234.108912:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828234.108918:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828234.108924:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828234.108931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327223680/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828234.108940:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828234.108943:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828234.108947:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327223680/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828234.108978:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828234.108982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828234.108985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828234.108987:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828234.108988:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828234.108991:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828234.108994:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2381
00000400:00000200:2.0:1787828234.108998:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828234.109001:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828234.109004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828234.109007:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109010:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.109012:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109014:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828234.109017:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828234.109018:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828234.109020:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828234.109022:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828234.109025:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828234.109027:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828234.109030:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109032:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.109033:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109034:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828234.109037:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828234.109038:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109040:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828234.109042:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828234.109047:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828234.109049:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109050:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828234.109055:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828234.109058:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109060:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828234.109064:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828234.109066:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828234.109068:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828234.109069:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109071:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828234.109072:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828234.109074:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828234.109077:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828234.109080:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828234.109085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327223808/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828234.109110:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828234.109113:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828234.109116:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327223808/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828234.109132:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828234.109137:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828234.109143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828234.109148:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828234.109152:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828234.109162:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828234.109176:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2381
00000400:00000200:2.0:1787828234.109186:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828234.109196:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828234.109200:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828234.109211:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109220:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.109236:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109246:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828234.109251:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828234.109253:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828234.109257:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828234.109258:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828234.109260:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828234.109271:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828234.109274:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109275:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.109276:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109277:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828234.109279:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828234.109280:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109282:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828234.109282:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828234.109285:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828234.109292:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109307:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828234.109315:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828234.109317:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109318:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828234.109322:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828234.109327:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828234.109338:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828234.109340:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109342:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828234.109343:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828234.109347:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828234.109353:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828234.109364:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828234.109372:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327223936/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828234.109379:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828234.109385:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828234.109393:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327223936/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828234.109422:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828234.109434:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828234.109446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828234.109464:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828234.109467:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828234.109472:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828234.109476:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2381
00000400:00000200:2.0:1787828234.109533:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828234.109543:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828234.109549:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828234.109559:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:3.1:1787828234.109567:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828234.109568:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.109575:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109577:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828234.109581:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.1:1787828234.109582:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000020:00000040:2.0:1787828234.109583:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828234.109590:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828234.109598:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828234.109606:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828234.109614:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828234.109619:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109622:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828234.109628:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109633:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828234.109636:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828234.109639:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828234.109643:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828234.109651:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828234.109703:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828234.109707:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109710:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828234.109713:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828234.109716:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109729:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828234.109735:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828234.109742:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828234.109750:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828234.109754:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828234.109774:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828234.109779:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828234.109783:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828234.109788:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828234.109793:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828234.109799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327224064/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828234.109807:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828234.109813:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828234.109827:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327224064/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828234.109855:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828234.109863:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2391)
00000800:00000200:3.1:1787828234.110269:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828234.110273:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828234.110497:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828234.110499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828234.110681:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828234.110683:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787828234.812828:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297054893, at 4297054912
00000100:00000010:2.0:1787828234.812844:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f540.
00000020:00000001:2.0:1787828235.964964:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828235.964971:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828235.964975:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828236.988896:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828236.988903:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828236.988908:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828239.037085:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828239.037094:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828239.037101:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828239.229580:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828239.229589:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828239.230366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828239.230369:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828239.230763:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828239.230766:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828239.231099:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828239.231119:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00000040:2.0:1787828239.932761:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297060013, at 4297060032
00000100:00000010:2.0:1787828239.932772:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f1e0.
00000020:00000001:2.0:1787828240.060808:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828240.060812:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828240.060815:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828242.108983:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828242.108988:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828242.108992:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828244.348757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828244.348765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828244.348767:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828244.348772:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828244.348777:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828244.348785:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2392
00000400:00000200:2.0:1787828244.348793:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828244.348800:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828244.348803:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828244.348807:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348811:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.348813:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348816:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828244.348820:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828244.348822:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828244.348824:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828244.348828:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828244.348831:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828244.348834:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828244.348837:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348839:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.348841:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348844:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828244.348847:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828244.348850:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348853:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828244.348856:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828244.348860:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828244.348863:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.348865:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828244.348867:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828244.348875:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828244.348878:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828244.348881:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828244.348883:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.348885:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828244.348887:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828244.348891:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828244.348897:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828244.348902:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828244.348909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327224704/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828244.348918:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828244.348921:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828244.348924:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327224704/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828244.348942:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828244.348945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828244.348948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828244.348949:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828244.348950:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828244.348952:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828244.348954:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2392
00000400:00000200:2.0:1787828244.348957:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828244.348961:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828244.348963:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828244.348965:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348966:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.348969:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.348972:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828244.348977:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828244.348979:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828244.348981:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828244.348984:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828244.348986:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828244.348990:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828244.348997:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349001:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.349004:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349007:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828244.349010:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828244.349015:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349019:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828244.349024:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828244.349030:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828244.349034:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349037:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828244.349042:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828244.349045:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349046:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828244.349048:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828244.349049:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828244.349051:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828244.349051:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349054:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828244.349055:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828244.349056:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828244.349058:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828244.349061:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828244.349066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327224832/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828244.349076:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828244.349080:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828244.349086:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327224832/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828244.349096:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828244.349099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828244.349105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828244.349107:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828244.349110:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828244.349115:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828244.349121:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2392
00000400:00000200:2.0:1787828244.349128:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828244.349135:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828244.349142:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828244.349149:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349154:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.349160:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349164:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828244.349169:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828244.349170:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828244.349172:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828244.349175:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828244.349177:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828244.349178:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828244.349182:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349185:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.349189:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349191:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828244.349193:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828244.349195:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349197:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828244.349201:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828244.349204:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828244.349207:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349211:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828244.349213:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828244.349217:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349220:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828244.349224:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828244.349228:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828244.349234:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828244.349237:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349239:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828244.349242:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828244.349244:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828244.349247:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828244.349252:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828244.349259:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327224960/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828244.349275:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828244.349278:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828244.349285:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327224960/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787828244.349291:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828244.349296:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:2.0:1787828244.349299:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828244.349304:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828244.349309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828244.349311:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828244.349312:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828244.349315:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828244.349320:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2392
00000400:00000200:2.0:1787828244.349327:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828244.349330:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828244.349332:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828244.349333:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349335:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.349338:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349341:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828244.349344:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828244.349346:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828244.349349:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828244.349350:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828244.349353:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828244.349357:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828244.349363:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349364:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828244.349365:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349370:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828244.349374:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828244.349377:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828244.349381:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828244.349383:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828244.349386:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828244.349391:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349396:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828244.349398:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828244.349402:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349405:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828244.349408:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828244.349413:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828244.349417:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828244.349420:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828244.349425:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828244.349426:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828244.349427:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828244.349431:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828244.349435:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828244.349440:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327225088/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828244.349450:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828244.349453:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828244.349460:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327225088/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828244.349472:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828244.349478:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2402)
00000800:00000200:3.1:1787828244.349723:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828244.349725:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828244.349924:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828244.349926:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828244.350132:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828244.350134:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000040:2.0:1787828245.052751:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297065132, at 4297065152
00000100:00000010:2.0:1787828245.052767:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f780.
00000020:00000001:2.0:1787828245.181007:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828245.181013:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828245.181017:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828248.252903:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828248.252908:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828248.252912:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828249.468763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828249.468774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828249.468776:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828249.468779:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828249.468783:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828249.468789:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2397
00000400:00000200:2.0:1787828249.468797:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828249.468803:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828249.468806:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828249.468809:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.468812:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.468815:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.468818:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828249.468822:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828249.468824:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828249.468828:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828249.468831:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828249.468833:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828249.468836:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828249.468838:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.468840:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.468842:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.468843:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828249.468845:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828249.468848:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828249.468850:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828249.468852:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828249.468856:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828249.468859:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.468861:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828249.468863:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828249.468871:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828249.468873:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828249.468877:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828249.468878:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.468881:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828249.468883:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828249.468888:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828249.468911:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828249.468917:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828249.468926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327225216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828249.468936:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828249.468939:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828249.468943:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327225216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828249.468962:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828249.468965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828249.468967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828249.468969:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828249.468970:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828249.468972:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828249.468974:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2397
00000400:00000200:2.0:1787828249.468977:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828249.468979:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828249.468981:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828249.468999:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469001:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.469004:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469006:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828249.469007:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828249.469008:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828249.469010:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828249.469013:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828249.469017:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828249.469019:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828249.469022:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469025:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.469028:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469031:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828249.469034:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828249.469036:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469038:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828249.469042:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828249.469047:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828249.469053:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469059:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828249.469067:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828249.469078:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469082:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828249.469092:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828249.469098:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828249.469101:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828249.469104:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469106:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828249.469112:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828249.469116:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828249.469123:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828249.469135:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828249.469143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327225344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828249.469161:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828249.469163:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828249.469172:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327225344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828249.469186:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828249.469189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828249.469192:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828249.469195:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828249.469196:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828249.469198:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828249.469201:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2397
00000400:00000200:2.0:1787828249.469209:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828249.469213:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828249.469221:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828249.469224:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469225:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.469229:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469235:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828249.469240:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828249.469260:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828249.469266:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828249.469272:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828249.469284:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828249.469293:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828249.469322:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469325:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.469334:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469339:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828249.469346:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828249.469354:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469368:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828249.469393:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828249.469402:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828249.469411:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469421:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828249.469426:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828249.469433:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469443:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828249.469454:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828249.469467:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828249.469478:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828249.469481:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469490:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828249.469499:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828249.469502:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828249.469511:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000800:00000200:3.1:1787828249.469518:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828249.469521:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828249.469523:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00100000:2.0:1787828249.469534:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327225472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828249.469553:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828249.469556:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828249.469563:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327225472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828249.469590:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828249.469598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828249.469605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828249.469609:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828249.469628:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828249.469649:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828249.469654:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2397
00000400:00000200:2.0:1787828249.469659:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828249.469663:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828249.469665:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828249.469667:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469669:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.469699:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469710:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828249.469720:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828249.469724:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828249.469727:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828249.469747:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828249.469750:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828249.469753:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828249.469756:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469758:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828249.469760:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469770:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828249.469772:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828249.469776:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828249.469781:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828249.469784:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828249.469789:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828249.469792:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469795:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828249.469801:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828249.469804:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469812:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828249.469817:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828249.469826:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828249.469832:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828249.469838:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828249.469848:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828249.469854:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828249.469857:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828249.469869:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828249.469878:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828249.469884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327225600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828249.469908:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828249.469911:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828249.469928:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327225600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828249.469955:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828249.469969:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2407)
00000800:00000200:3.1:1787828249.470211:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828249.470213:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828249.470418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828249.470420:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828249.470601:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828249.470603:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1787828250.172754:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297070253, at 4297070272
00000100:00000010:2.0:1787828250.172784:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f180.
00000020:00000001:2.0:1787828251.324818:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828251.324823:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828251.324827:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828254.396929:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828254.396938:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828254.396945:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828254.588726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828254.588736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828254.588739:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828254.588743:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828254.588747:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828254.588754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2402
00000400:00000200:2.0:1787828254.588763:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828254.588769:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828254.588772:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828254.588775:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588779:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.588781:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588784:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828254.588788:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828254.588790:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828254.588792:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828254.588796:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828254.588799:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828254.588801:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828254.588804:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588806:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.588808:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588810:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828254.588813:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828254.588816:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588819:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828254.588821:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828254.588825:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828254.588828:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.588830:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828254.588832:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828254.588841:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787828254.588844:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828254.588848:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828254.588850:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.588853:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828254.588855:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828254.588859:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828254.588866:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828254.588872:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828254.588881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327225728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828254.588898:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828254.588902:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828254.588906:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327225728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828254.588926:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828254.588929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828254.588932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828254.588934:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828254.588935:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828254.588938:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828254.588941:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2402
00000400:00000200:2.0:1787828254.588945:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828254.588948:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828254.588951:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828254.588954:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588956:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.588959:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.588965:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828254.588969:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828254.588973:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828254.588976:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828254.588979:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828254.588985:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828254.588990:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828254.588998:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589010:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.589016:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589021:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828254.589025:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828254.589028:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589035:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828254.589037:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828254.589044:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828254.589049:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589054:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828254.589058:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828254.589063:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589067:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828254.589075:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828254.589079:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828254.589084:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828254.589085:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589090:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828254.589093:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828254.589097:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828254.589103:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828254.589110:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828254.589121:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327225856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828254.589137:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828254.589142:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828254.589149:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327225856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828254.589165:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828254.589172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828254.589186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828254.589191:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828254.589195:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828254.589201:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828254.589207:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2402
00000400:00000200:2.0:1787828254.589214:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828254.589222:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828254.589227:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1787828254.589227:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828254.589232:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787828254.589234:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589239:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.589246:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589250:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828254.589258:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828254.589261:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828254.589266:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828254.589270:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828254.589275:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828254.589281:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828254.589288:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589291:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.589297:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589301:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828254.589306:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828254.589312:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589316:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828254.589320:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828254.589323:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828254.589328:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589331:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828254.589334:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828254.589338:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589342:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828254.589347:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828254.589353:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828254.589358:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828254.589361:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589364:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828254.589368:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828254.589371:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828254.589380:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828254.589386:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828254.589396:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327225984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828254.589423:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828254.589428:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828254.589438:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327225984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828254.589454:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828254.589460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828254.589469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828254.589473:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828254.589476:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828254.589491:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828254.589498:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2402
00000400:00000200:2.0:1787828254.589507:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828254.589515:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828254.589520:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828254.589527:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589532:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.589538:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589543:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828254.589549:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828254.589554:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828254.589558:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828254.589562:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828254.589568:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828254.589590:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828254.589600:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589605:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828254.589612:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589615:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828254.589619:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828254.589625:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828254.589630:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828254.589634:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828254.589637:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828254.589641:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589646:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828254.589648:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828254.589653:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589673:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828254.589677:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828254.589679:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828254.589681:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828254.589683:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828254.589685:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828254.589686:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828254.589690:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828254.589697:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828254.589705:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828254.589715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327226112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828254.589731:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828254.589736:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828254.589745:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327226112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828254.589775:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828254.589784:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2412)
00000800:00000200:3.1:1787828254.589866:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828254.589867:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828254.590130:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828254.590132:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828254.590396:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828254.590408:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787828255.292805:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297075372, at 4297075392
00000100:00000010:2.0:1787828255.292822:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f060.
00000020:00000001:2.0:1787828257.468880:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828257.468885:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828257.468889:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828259.708794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828259.708808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828259.708811:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828259.708816:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828259.708822:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828259.708832:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2407
00000400:00000200:2.0:1787828259.708843:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828259.708851:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828259.708856:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828259.708860:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.708864:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.708868:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.708871:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828259.708877:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828259.708880:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828259.708884:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828259.708889:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828259.708892:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828259.708896:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828259.708901:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.708904:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.708908:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.708911:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828259.708915:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828259.708919:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828259.708924:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828259.708927:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828259.708932:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828259.708936:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.708939:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828259.708941:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828259.708953:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828259.708957:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828259.708962:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828259.708965:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.708968:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828259.708972:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828259.708977:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828259.708986:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828259.708994:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828259.709005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327226240/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828259.709019:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828259.709025:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828259.709030:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327226240/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828259.709060:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828259.709065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828259.709069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828259.709071:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828259.709074:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828259.709077:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828259.709080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2407
00000400:00000200:2.0:1787828259.709085:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828259.709088:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828259.709091:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828259.709094:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709098:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.709101:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709103:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828259.709106:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828259.709109:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828259.709111:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828259.709114:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828259.709117:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828259.709120:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828259.709125:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709128:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.709131:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709134:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828259.709137:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828259.709142:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709152:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828259.709155:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828259.709165:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828259.709171:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.709175:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828259.709181:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828259.709192:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.709200:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828259.709210:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828259.709216:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828259.709224:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828259.709229:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.709236:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828259.709238:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828259.709246:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828259.709259:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828259.709271:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828259.709292:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327226368/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828259.709313:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828259.709318:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828259.709329:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327226368/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828259.709361:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828259.709367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828259.709373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828259.709376:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828259.709379:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828259.709391:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828259.709399:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2407
00000400:00000200:2.0:1787828259.709409:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828259.709416:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828259.709420:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828259.709429:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709437:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.709444:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709451:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828259.709460:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828259.709464:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828259.709470:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828259.709477:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828259.709485:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828259.709489:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828259.709501:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709507:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.709517:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709523:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828259.709530:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828259.709565:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709572:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828259.709581:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828259.709591:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828259.709600:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.709606:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828259.709612:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828259.709622:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.709630:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828259.709641:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828259.709650:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828259.709703:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828259.709708:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.709713:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828259.709718:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828259.709724:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828259.709735:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828259.709746:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828259.709763:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327226496/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828259.709783:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828259.709790:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828259.709799:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327226496/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828259.709818:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828259.709825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828259.709831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828259.709840:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828259.709845:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828259.709855:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828259.709863:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2407
00000800:00000200:3.1:1787828259.709872:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:2.0:1787828259.709874:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:3.1:1787828259.709884:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000400:00000200:2.0:1787828259.709887:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828259.709897:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828259.709908:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709914:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.709922:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709925:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828259.709929:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828259.709936:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828259.709943:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828259.709947:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828259.709955:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828259.709959:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828259.709971:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709977:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828259.709984:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828259.709991:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828259.709996:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828259.710010:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828259.710020:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828259.710028:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828259.710036:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828259.710043:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.710051:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828259.710055:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828259.710066:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.710074:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828259.710084:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828259.710093:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828259.710102:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828259.710105:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828259.710109:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828259.710114:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828259.710119:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828259.710130:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828259.710137:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828259.710152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327226624/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828259.710167:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828259.710174:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828259.710181:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327226624/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828259.710208:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828259.710217:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2417)
00000800:00000200:3.1:1787828259.710703:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828259.710707:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828259.711148:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828259.711151:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828259.711585:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828259.711588:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828260.412784:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297080493, at 4297080512
00000100:00000010:2.0:1787828260.412819:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fde0.
00000020:00000001:2.0:1787828260.540906:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828260.540910:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828260.540914:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828263.612879:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828263.612884:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828263.612888:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828264.830159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828264.830181:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828264.831366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828264.831370:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828264.831821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828264.831824:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828264.832147:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828264.832150:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828265.532840:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297085613, at 4297085632
00000100:00000010:2.0:1787828265.532863:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f300.
00000800:00000200:3.1:1787828269.949871:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828269.949881:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828269.950953:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828269.950956:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828269.951344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828269.951349:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828269.951847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828269.951868:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828270.652722:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297090733, at 4297090752
00000100:00000010:2.0:1787828270.652740:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f840.
00000800:00000200:3.1:1787828275.069768:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828275.069782:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828275.070579:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828275.070583:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828275.071083:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828275.071088:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828275.071496:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828275.071500:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828275.772770:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297095853, at 4297095872
00000100:00000010:2.0:1787828275.772791:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fae0.
00000800:00000200:3.1:1787828280.189457:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828280.189461:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828280.190223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828280.190228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828280.190553:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828280.190557:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828280.191058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828280.191060:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1787828280.892718:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297100972, at 4297100992
00000100:00000010:2.0:1787828280.892733:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fb40.
00000800:00000200:3.1:1787828285.309295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828285.309299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828285.309817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828285.309818:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828285.310023:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828285.310024:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828285.310216:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828285.310231:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000040:2.0:1787828286.012724:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297106092, at 4297106112
00000100:00000010:2.0:1787828286.012738:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd80.
00000100:00080000:2.0:1787828290.428813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828290.428824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828290.428826:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828290.428830:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828290.428835:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828290.428843:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2438
00000400:00000200:2.0:1787828290.428851:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828290.428858:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828290.428861:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828290.428864:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.428867:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.428869:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.428871:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828290.428874:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828290.428876:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828290.428878:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828290.428882:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828290.428883:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828290.428885:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828290.428888:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.428889:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.428891:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.428892:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828290.428895:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828290.428897:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828290.428899:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828290.428902:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828290.428906:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828290.428908:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.428910:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828290.428913:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828290.428923:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828290.428926:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828290.428931:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828290.428932:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.428935:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828290.428938:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828290.428943:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828290.428953:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828290.428960:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828290.428969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327229312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828290.428978:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828290.428983:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828290.428987:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327229312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828290.429007:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828290.429010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828290.429013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828290.429014:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828290.429015:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828290.429017:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828290.429019:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2438
00000400:00000200:2.0:1787828290.429021:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828290.429023:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828290.429025:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828290.429028:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429030:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.429033:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429035:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828290.429037:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828290.429038:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828290.429039:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828290.429041:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828290.429044:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828290.429047:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828290.429054:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429061:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.429067:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429075:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828290.429081:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828290.429088:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429096:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828290.429101:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828290.429109:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828290.429124:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429127:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828290.429133:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828290.429138:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429141:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828290.429145:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828290.429149:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828290.429151:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828290.429155:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429157:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828290.429158:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828290.429160:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828290.429167:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828290.429172:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828290.429179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327229440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828290.429203:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828290.429207:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828290.429216:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327229440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828290.429230:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828290.429235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828290.429241:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828290.429242:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828290.429244:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828290.429247:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828290.429254:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2438
00000400:00000200:2.0:1787828290.429299:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828290.429308:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828290.429312:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828290.429315:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429318:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.429320:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429326:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828290.429347:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828290.429350:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:3.1:1787828290.429351:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787828290.429355:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828290.429360:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828290.429363:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828290.429365:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828290.429380:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429384:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.429388:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429394:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828290.429399:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828290.429406:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429411:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828290.429416:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828290.429419:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828290.429424:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429428:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828290.429431:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828290.429437:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429443:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828290.429446:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828290.429447:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828290.429452:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828290.429455:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429459:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828290.429463:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828290.429468:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828290.429477:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828290.429481:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828290.429492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327229568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828290.429504:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828290.429508:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828290.429516:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327229568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828290.429531:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828290.429537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828290.429542:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828290.429545:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828290.429548:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828290.429554:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828290.429561:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2438
00000400:00000200:2.0:1787828290.429566:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828290.429577:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828290.429580:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828290.429582:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429586:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.429590:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429594:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828290.429600:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828290.429603:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828290.429606:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828290.429610:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828290.429613:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828290.429617:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828290.429623:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429626:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828290.429629:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429631:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828290.429635:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828290.429641:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828290.429646:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828290.429648:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828290.429651:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828290.429676:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429679:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828290.429682:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828290.429685:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429687:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828290.429691:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828290.429694:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828290.429700:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828290.429704:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828290.429707:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828290.429711:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828290.429715:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828290.429722:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828290.429742:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828290.429754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327229696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828290.429771:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828290.429777:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828290.429778:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828290.429784:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327229696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828290.429796:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828290.429804:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2448)
00000800:00000200:3.1:1787828290.429981:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828290.430127:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828291.132719:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297111212, at 4297111232
00000100:00000010:2.0:1787828291.132736:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f720.
00000800:00000200:3.1:1787828295.549994:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828295.549998:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828295.550432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828295.550435:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828295.550806:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828295.550809:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828295.551122:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828295.551125:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1787828296.252701:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297116333, at 4297116352
00000100:00000010:2.0:1787828296.252713:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f4e0.
00000100:00080000:2.0:1787828300.668783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828300.668792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828300.668795:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828300.668798:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828300.668803:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828300.668809:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2448
00000400:00000200:2.0:1787828300.668818:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828300.668824:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828300.668827:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828300.668830:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.668833:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.668835:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.668837:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828300.668842:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828300.668843:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828300.668846:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828300.668849:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828300.668851:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828300.668854:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828300.668857:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.668859:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.668861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.668863:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828300.668865:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828300.668869:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828300.668871:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828300.668874:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828300.668877:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828300.668880:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.668882:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828300.668884:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828300.668893:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828300.668896:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828300.668900:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828300.668902:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.668904:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828300.668907:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828300.668911:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828300.668919:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828300.668924:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828300.668933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327230336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828300.668942:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828300.668947:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828300.668951:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327230336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828300.668975:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828300.668978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828300.668981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828300.668983:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828300.668984:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828300.668987:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828300.668989:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2448
00000400:00000200:2.0:1787828300.668992:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828300.668995:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828300.668998:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828300.669001:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669004:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.669006:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669009:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828300.669011:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828300.669012:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828300.669014:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828300.669019:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828300.669024:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828300.669029:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828300.669036:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669041:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.669046:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669053:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828300.669058:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828300.669065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669071:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828300.669076:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828300.669084:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828300.669087:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669091:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828300.669097:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828300.669100:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669103:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828300.669109:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828300.669113:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828300.669116:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828300.669118:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669121:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828300.669122:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828300.669124:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828300.669131:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828300.669134:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828300.669142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327230464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828300.669153:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828300.669157:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828300.669164:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327230464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828300.669179:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828300.669183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828300.669187:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828300.669190:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828300.669192:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828300.669195:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828300.669197:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2448
00000400:00000200:2.0:1787828300.669241:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828300.669247:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828300.669249:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828300.669254:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669269:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.669273:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669275:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828300.669279:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828300.669281:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828300.669283:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828300.669286:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828300.669289:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828300.669292:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:3.1:1787828300.669293:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828300.669297:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:3.1:1787828300.669299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787828300.669300:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.669303:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669306:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828300.669310:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828300.669316:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669319:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828300.669321:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828300.669323:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828300.669325:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669328:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828300.669332:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828300.669338:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669339:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828300.669342:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828300.669348:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828300.669351:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828300.669354:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669356:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828300.669358:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828300.669361:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828300.669367:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828300.669371:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828300.669379:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327230592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828300.669397:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828300.669401:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828300.669406:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327230592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828300.669417:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828300.669421:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828300.669426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828300.669428:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828300.669430:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828300.669434:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828300.669436:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2448
00000400:00000200:2.0:1787828300.669441:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828300.669446:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828300.669450:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828300.669455:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669457:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.669461:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669464:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828300.669470:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828300.669472:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828300.669475:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828300.669478:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828300.669481:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828300.669484:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828300.669491:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669495:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828300.669500:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669502:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828300.669505:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828300.669510:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828300.669514:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828300.669517:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828300.669521:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828300.669523:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669526:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828300.669528:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828300.669530:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669533:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828300.669537:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828300.669540:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828300.669544:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828300.669546:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828300.669548:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828300.669551:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828300.669553:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828300.669557:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828300.669562:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828300.669569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327230720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828300.669580:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828300.669583:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828300.669589:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327230720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828300.669600:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828300.669606:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2458)
00000800:00000200:3.1:1787828300.669850:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828300.669852:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828300.670038:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828300.670040:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828300.670217:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828300.670219:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787828301.372739:0:16211:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297121452, at 4297121472
00000100:00000010:2.0:1787828301.372754:0:16211:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f900.
00000800:00000200:3.1:1787828305.789379:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828305.789383:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828305.789822:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828305.789824:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828305.790039:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828305.790041:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828305.790218:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828305.790236:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000040:2.0:1787828306.492729:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297126572, at 4297126592
00000100:00000010:2.0:1787828306.492748:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3ff00.
00000100:00080000:2.0:1787828310.908789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828310.908800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828310.908803:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828310.908807:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828310.908815:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828310.908823:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2458
00000400:00000200:2.0:1787828310.908832:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828310.908838:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828310.908842:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828310.908844:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.908846:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.908849:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.908850:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828310.908855:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828310.908857:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828310.908859:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828310.908863:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828310.908865:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828310.908867:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828310.908870:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.908872:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.908874:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.908877:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828310.908885:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828310.908889:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828310.908892:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828310.908896:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828310.908912:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828310.908916:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.908920:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828310.908923:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828310.908936:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787828310.908940:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828310.908945:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828310.908947:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.908950:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828310.908953:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828310.908958:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828310.908967:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828310.908975:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828310.908987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327231360/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828310.908999:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828310.909005:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828310.909022:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327231360/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828310.909048:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828310.909051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828310.909053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828310.909055:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828310.909056:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828310.909059:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828310.909061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2458
00000400:00000200:2.0:1787828310.909064:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828310.909067:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828310.909068:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828310.909071:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909073:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.909075:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909087:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828310.909089:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828310.909091:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828310.909093:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828310.909095:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828310.909097:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828310.909098:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828310.909101:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909104:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.909107:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909111:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828310.909115:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828310.909119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909121:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828310.909125:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828310.909133:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828310.909138:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909141:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828310.909146:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828310.909152:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909157:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828310.909163:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828310.909167:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828310.909170:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828310.909172:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909176:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828310.909180:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828310.909182:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828310.909221:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828310.909228:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828310.909236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327231488/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828310.909248:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828310.909253:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828310.909259:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327231488/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828310.909283:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828310.909288:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828310.909292:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828310.909295:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828310.909298:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828310.909301:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828310.909306:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2458
00000400:00000200:2.0:1787828310.909313:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828310.909315:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828310.909317:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828310.909322:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909326:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.909330:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909336:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828310.909339:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828310.909340:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828310.909343:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828310.909356:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828310.909359:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828310.909363:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828310.909369:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909373:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.909378:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909381:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828310.909385:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828310.909390:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909394:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828310.909396:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828310.909399:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828310.909402:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909404:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828310.909406:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828310.909408:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909412:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828310.909419:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828310.909424:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828310.909429:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828310.909430:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909432:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828310.909433:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828310.909437:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828310.909441:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828310.909445:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828310.909452:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327231616/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787828310.909458:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828310.909465:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787828310.909465:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787828310.909468:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828310.909475:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327231616/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828310.909485:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828310.909488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828310.909493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828310.909497:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828310.909500:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828310.909503:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828310.909508:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2458
00000400:00000200:2.0:1787828310.909522:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828310.909527:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828310.909530:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828310.909535:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909538:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.909542:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909546:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828310.909550:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828310.909553:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828310.909554:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828310.909556:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828310.909559:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828310.909562:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828310.909568:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909572:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828310.909575:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909578:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828310.909580:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828310.909589:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828310.909593:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828310.909596:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828310.909598:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828310.909599:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909601:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828310.909605:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828310.909609:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909613:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828310.909617:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828310.909619:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828310.909621:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828310.909625:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828310.909626:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828310.909628:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828310.909630:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828310.909634:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828310.909638:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828310.909645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327231744/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828310.909649:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828310.909653:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828310.909678:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327231744/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828310.909685:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828310.909693:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2468)
00000800:00000200:3.1:1787828310.909975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828310.909976:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828310.910176:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828310.910177:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828310.910397:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828310.910399:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1787828311.612727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297131692, at 4297131712
00000100:00000010:2.0:1787828311.612756:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f660.
00000800:00000200:3.1:1787828316.029560:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828316.029567:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828316.030305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828316.030308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828316.030562:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828316.030564:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828316.030935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828316.030937:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1787828316.732842:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297136813, at 4297136832
00000100:00000010:2.0:1787828316.732901:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fba0.
00000800:00000200:3.1:1787828321.149875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828321.149914:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828321.150845:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828321.150849:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828321.151221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828321.151223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828321.151463:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828321.151466:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000040:2.0:1787828321.852891:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297141933, at 4297141952
00000100:00000010:2.0:1787828321.852925:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f3c0.
00000800:00000200:3.1:1787828326.269535:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828326.269542:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828326.270261:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828326.270263:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828326.270479:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828326.270480:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828326.270704:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828326.270706:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828326.972719:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297147053, at 4297147072
00000100:00000010:2.0:1787828326.972733:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f480.
00000100:00080000:2.0:1787828331.388823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828331.388881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828331.388884:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828331.388888:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828331.388893:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828331.388901:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2479
00000400:00000200:2.0:1787828331.388910:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828331.388931:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828331.388935:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828331.388939:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.388943:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.388946:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.388949:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828331.388953:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828331.388955:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828331.388958:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828331.388962:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828331.388965:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828331.388968:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828331.388971:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.388973:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.388975:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.388978:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828331.388981:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828331.388985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828331.388988:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828331.388990:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828331.388995:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828331.388998:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389000:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828331.389003:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828331.389013:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828331.389017:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828331.389021:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828331.389023:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389026:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828331.389029:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828331.389034:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828331.389041:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828331.389060:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828331.389070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327233408/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828331.389090:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828331.389095:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828331.389100:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327233408/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828331.389124:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828331.389128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828331.389131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828331.389133:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828331.389135:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828331.389138:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828331.389141:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2479
00000400:00000200:2.0:1787828331.389145:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828331.389148:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828331.389151:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828331.389155:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389158:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.389161:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389163:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828331.389166:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828331.389167:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828331.389169:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828331.389172:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828331.389177:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828331.389180:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828331.389186:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389190:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.389196:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389202:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828331.389209:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828331.389216:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389221:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828331.389225:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828331.389233:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828331.389238:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389243:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828331.389249:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828331.389257:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389260:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828331.389265:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828331.389267:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828331.389271:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828331.389274:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389279:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828331.389283:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828331.389287:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828331.389296:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828331.389302:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828331.389312:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327233536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828331.389327:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828331.389331:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828331.389341:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327233536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828331.389358:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828331.389364:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828331.389371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828331.389375:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828331.389377:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828331.389382:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828331.389387:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2479
00000400:00000200:2.0:1787828331.389442:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828331.389451:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828331.389456:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828331.389462:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389468:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.389470:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389474:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828331.389479:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828331.389482:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:3.1:1787828331.389484:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787828331.389485:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1787828331.389490:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000020:00000040:2.0:1787828331.389491:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828331.389495:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828331.389499:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828331.389512:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389515:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.389519:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389524:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828331.389529:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828331.389536:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389540:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828331.389543:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828331.389548:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828331.389553:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389556:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828331.389561:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828331.389566:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389570:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828331.389578:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828331.389584:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828331.389589:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828331.389594:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389596:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828331.389599:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828331.389602:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828331.389607:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828331.389613:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828331.389620:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327233664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828331.389628:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828331.389631:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828331.389637:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327233664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828331.389647:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828331.389652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828331.389679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828331.389680:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828331.389682:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828331.389684:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828331.389686:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2479
00000400:00000200:2.0:1787828331.389691:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828331.389695:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828331.389698:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828331.389701:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389705:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.389712:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389717:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828331.389722:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828331.389724:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828331.389725:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828331.389729:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828331.389732:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828331.389738:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828331.389746:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389751:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828331.389755:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389759:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828331.389762:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828331.389767:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828331.389772:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828331.389776:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828331.389780:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828331.389784:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389788:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828331.389791:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828331.389795:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389799:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828331.389804:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828331.389806:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828331.389811:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828331.389813:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828331.389816:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828331.389819:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828331.389822:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828331.389826:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828331.389830:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828331.389834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327233792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828331.389843:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828331.389846:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828331.389854:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327233792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828331.389880:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828331.389886:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2489)
00000800:00000200:3.1:1787828331.390119:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828331.390121:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828331.390357:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828331.390359:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828331.390545:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828331.390547:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828332.092718:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297152172, at 4297152192
00000100:00000010:2.0:1787828332.092732:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f0c0.
00000100:00080000:2.0:1787828336.508794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828336.508806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828336.508811:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828336.508817:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828336.508826:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828336.508837:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2484
00000400:00000200:2.0:1787828336.508851:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828336.508863:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828336.508869:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828336.508876:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.508882:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.508887:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.508893:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828336.508901:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828336.508905:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828336.508910:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828336.508915:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828336.508918:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828336.508922:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828336.508926:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.508929:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.508932:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.508934:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828336.508938:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828336.508943:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828336.508946:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828336.508949:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828336.508955:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828336.508959:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.508962:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828336.508965:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828336.508977:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828336.508982:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828336.508987:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828336.508989:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.508993:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828336.508996:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828336.509002:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828336.509041:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828336.509051:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828336.509064:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327233920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828336.509079:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828336.509084:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828336.509090:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327233920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828336.509123:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828336.509128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828336.509132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828336.509134:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828336.509136:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828336.509140:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828336.509143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2484
00000400:00000200:2.0:1787828336.509148:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828336.509152:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828336.509155:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828336.509160:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509164:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.509168:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509171:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828336.509175:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828336.509176:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828336.509179:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828336.509183:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828336.509190:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828336.509199:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828336.509209:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509216:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.509223:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509232:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828336.509241:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828336.509251:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509257:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828336.509262:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828336.509271:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828336.509274:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.509279:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828336.509285:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828336.509292:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.509298:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828336.509303:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828336.509310:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828336.509316:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828336.509321:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.509329:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828336.509332:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828336.509338:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828336.509347:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828336.509358:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828336.509371:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327234048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828336.509398:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828336.509403:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828336.509410:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327234048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828336.509429:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828336.509437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828336.509449:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828336.509454:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828336.509460:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828336.509469:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828336.509481:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2484
00000400:00000200:2.0:1787828336.509497:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828336.509510:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828336.509519:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828336.509532:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509541:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.509549:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509558:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828336.509569:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828336.509573:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828336.509579:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828336.509584:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828336.509590:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828336.509596:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828336.509610:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509618:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.509628:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509637:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828336.509648:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828336.509705:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828336.509712:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828336.509717:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828336.509724:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828336.509730:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.509740:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828336.509750:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828336.509760:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.509765:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828336.509774:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828336.509781:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828336.509786:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828336.509791:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.509799:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828336.509801:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828336.509805:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828336.509816:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828336.509827:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828336.509842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327234176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828336.509877:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787828336.509887:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828336.509888:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828336.509902:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828336.509905:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327234176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828336.509929:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828336.509937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828336.509949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828336.509952:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828336.509954:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828336.509959:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828336.509969:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2484
00000400:00000200:2.0:1787828336.509983:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828336.509990:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828336.509992:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828336.509996:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.510000:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.510033:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.510042:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828336.510050:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828336.510053:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828336.510057:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828336.510062:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828336.510071:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828336.510079:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828336.510092:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.510100:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828336.510106:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828336.510114:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828336.510124:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828336.510131:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828336.510141:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828336.510145:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828336.510153:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828336.510162:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.510183:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828336.510190:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828336.510203:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.510210:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828336.510217:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828336.510225:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828336.510230:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828336.510235:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828336.510240:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828336.510244:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828336.510250:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828336.510260:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828336.510270:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828336.510287:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327234304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828336.510305:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828336.510312:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828336.510326:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327234304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828336.510345:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828336.510354:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2494)
00000800:00000200:3.1:1787828336.510825:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828336.510830:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828336.511307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828336.511310:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828336.511749:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828336.511755:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828337.212821:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297157293, at 4297157312
00000100:00000010:2.0:1787828337.212841:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fea0.
00000020:00000001:2.0:1787828337.340990:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828337.340996:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828337.341023:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828340.412906:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828340.412912:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828340.412917:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828341.628779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828341.628792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828341.628806:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828341.628811:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828341.628817:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828341.628827:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2489
00000400:00000200:2.0:1787828341.628839:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828341.628850:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828341.628854:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828341.628858:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.628862:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.628865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.628867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828341.628872:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828341.628874:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828341.628877:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828341.628880:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828341.628883:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828341.628885:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828341.628888:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.628890:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.628892:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.628894:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828341.628897:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828341.628900:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828341.628903:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828341.628905:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828341.628909:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828341.628911:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.628913:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828341.628916:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828341.628924:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828341.628928:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828341.628944:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828341.628946:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.628949:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828341.628952:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828341.628957:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828341.628964:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828341.628991:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828341.629013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327234432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828341.629031:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828341.629035:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828341.629040:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327234432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828341.629073:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828341.629077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828341.629080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828341.629082:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828341.629084:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828341.629087:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828341.629089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2489
00000400:00000200:2.0:1787828341.629093:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828341.629097:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828341.629099:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828341.629103:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629106:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.629109:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629111:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828341.629114:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828341.629116:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828341.629117:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828341.629119:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828341.629122:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828341.629128:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828341.629134:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629140:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.629147:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629151:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828341.629158:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828341.629165:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629173:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828341.629175:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828341.629182:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828341.629189:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629193:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828341.629199:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828341.629207:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629213:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828341.629220:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828341.629226:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828341.629235:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828341.629238:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629242:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828341.629248:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828341.629251:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828341.629258:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828341.629265:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828341.629283:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327234560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828341.629303:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828341.629311:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828341.629331:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327234560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828341.629384:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828341.629396:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828341.629408:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828341.629411:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828341.629413:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828341.629416:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828341.629421:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2489
00000400:00000200:2.0:1787828341.629479:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828341.629489:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828341.629494:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828341.629499:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629502:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.629508:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629513:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828341.629518:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828341.629519:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828341.629522:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828341.629527:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828341.629532:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828341.629536:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:3.1:1787828341.629539:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828341.629543:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629549:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1787828341.629550:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828341.629552:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629559:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828341.629563:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828341.629571:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629578:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828341.629585:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828341.629589:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828341.629593:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629605:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828341.629608:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828341.629614:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629619:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828341.629624:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828341.629630:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828341.629636:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828341.629641:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629644:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828341.629648:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828341.629652:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828341.629686:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828341.629691:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828341.629703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327234688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828341.629718:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828341.629725:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828341.629738:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327234688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828341.629751:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828341.629757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828341.629763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828341.629768:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828341.629770:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828341.629774:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828341.629780:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2489
00000400:00000200:2.0:1787828341.629785:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828341.629790:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828341.629795:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828341.629798:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629802:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.629808:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629812:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828341.629815:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828341.629817:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828341.629823:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828341.629826:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828341.629829:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828341.629838:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828341.629843:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629848:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828341.629856:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629863:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828341.629869:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828341.629879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828341.629891:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828341.629898:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828341.629907:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828341.629917:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629926:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828341.629933:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828341.629944:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.629952:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828341.629966:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828341.629994:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828341.630005:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828341.630027:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828341.630036:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828341.630045:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828341.630054:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828341.630071:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828341.630088:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828341.630112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327234816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828341.630162:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828341.630174:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828341.630203:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327234816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828341.630255:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828341.630270:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2499)
00000800:00000200:3.1:1787828341.630503:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828341.630507:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828341.630890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828341.630893:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828341.631280:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828341.631285:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828342.332743:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297162413, at 4297162432
00000100:00000010:2.0:1787828342.332759:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f2a0.
00000020:00000001:2.0:1787828343.485003:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828343.485021:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828343.485027:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828346.556878:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828346.556883:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828346.556886:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828346.749772:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828346.749776:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828346.750329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828346.750332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828346.750643:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828346.750646:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828346.750998:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828346.751001:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1787828347.452755:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297167533, at 4297167552
00000100:00000010:2.0:1787828347.452773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fe40.
00000020:00000001:2.0:1787828349.628882:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828349.628886:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828349.628889:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828351.868870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828351.868882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828351.868886:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828351.868890:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828351.868895:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828351.868914:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2499
00000400:00000200:2.0:1787828351.868926:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828351.868933:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828351.868937:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828351.868959:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.868963:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.868967:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.868970:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828351.868976:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828351.868979:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828351.868983:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828351.868988:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828351.868991:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828351.868996:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828351.869000:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869003:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.869005:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869008:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828351.869012:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828351.869017:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869020:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828351.869024:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828351.869029:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828351.869033:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869036:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828351.869040:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828351.869064:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787828351.869069:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828351.869076:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828351.869078:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869082:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828351.869086:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828351.869093:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828351.869105:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828351.869132:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828351.869146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327235456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828351.869159:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828351.869165:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828351.869171:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327235456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828351.869211:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828351.869214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828351.869217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828351.869219:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828351.869221:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828351.869235:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828351.869237:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2499
00000400:00000200:2.0:1787828351.869240:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828351.869243:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828351.869244:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828351.869251:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869253:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.869255:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869258:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828351.869260:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828351.869261:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828351.869262:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828351.869264:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828351.869266:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828351.869268:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828351.869270:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869272:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.869273:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869275:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828351.869278:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828351.869281:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869283:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828351.869285:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828351.869291:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828351.869303:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869305:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828351.869311:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828351.869314:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869316:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828351.869321:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828351.869323:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828351.869326:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828351.869329:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869333:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828351.869336:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828351.869339:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828351.869346:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828351.869354:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828351.869368:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327235584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828351.869415:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828351.869429:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828351.869446:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327235584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828351.869466:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828351.869473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828351.869483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828351.869490:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828351.869496:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828351.869501:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828351.869511:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2499
00000400:00000200:2.0:1787828351.869521:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828351.869524:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828351.869528:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828351.869541:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869546:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.869555:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869560:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828351.869572:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828351.869574:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828351.869578:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828351.869585:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828351.869587:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828351.869590:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828351.869604:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869621:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.869630:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869646:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828351.869692:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828351.869697:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828351.869699:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828351.869701:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828351.869704:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828351.869707:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869709:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828351.869712:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828351.869715:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869717:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828351.869721:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828351.869723:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828351.869727:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828351.869732:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.869738:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828351.869742:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828351.869744:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828351.869751:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828351.869757:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828351.869784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327235712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828351.869813:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828351.869818:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828351.869826:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327235712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828351.869852:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828351.869861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828351.869873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828351.869877:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828351.869886:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828351.869897:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828351.869959:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2499
00000400:00000200:2.0:1787828351.870011:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828351.870026:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828351.870032:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828351.870045:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.870051:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1787828351.870055:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828351.870062:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:3.1:1787828351.870064:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1787828351.870070:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828351.870074:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828351.870078:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828351.870085:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828351.870097:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828351.870102:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828351.870115:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828351.870148:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.870160:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828351.870166:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828351.870170:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828351.870176:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828351.870186:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828351.870191:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828351.870205:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828351.870208:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828351.870216:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.870222:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828351.870226:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828351.870234:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.870238:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828351.870243:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828351.870246:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828351.870256:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828351.870258:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828351.870261:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828351.870267:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828351.870270:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828351.870278:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828351.870287:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828351.870298:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327235840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828351.870314:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828351.870318:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828351.870322:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327235840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828351.870341:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828351.870351:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2509)
00000800:00000200:3.1:1787828351.870761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828351.870765:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828351.871195:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828351.871200:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828351.871548:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828351.871551:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000040:2.0:1787828352.572817:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297172653, at 4297172672
00000100:00000010:2.0:1787828352.572846:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fcc0.
00000020:00000001:2.0:1787828352.701093:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828352.701099:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828352.701106:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828355.772950:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828355.772957:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828355.772963:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828356.989310:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828356.989322:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828356.989330:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828356.989347:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.989376:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828356.989383:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.989392:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828356.989398:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828356.989407:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828356.989420:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327235968:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828356.989431:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828356.989440:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828356.989447:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.989456:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828356.989468:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828356.989476:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828356.989492:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828356.989500:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828356.989517:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828356.989532:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327235968, portal 25
00000100:00000001:2.0:1787828356.989540:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828356.989545:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:2.0:1787828356.989564:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.989595:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828356.989607:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828356.989611:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6908.
00000100:00000200:2.0:1787828356.989614:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327235968, offset 0
00000400:00000200:2.0:1787828356.989629:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828356.989648:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.989706:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828356.989723:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828356.989737:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828356.989752:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828356.989764:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828356.989778:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39615:39615:256:4294967295] 192.168.203.116@tcp LPNI seq info [39615:39615:8:4294967295]
00000400:00000200:2.0:1787828356.989791:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.989834:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828356.989854:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828356.989886:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37200.
00000800:00000200:2.0:1787828356.989903:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828356.989933:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828356.989962:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828356.989975:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828356.989981:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297177773
00000100:00000010:2.0:1787828356.989991:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828356.990006:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828356.990022:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297177773
00000100:00000010:2.0:1787828356.990031:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9900.
00000100:00000001:2.0:1787828356.990042:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828356.990050:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828356.990060:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990068:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990077:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828356.990089:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828356.990095:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787828356.990107:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828356.990111:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828356.990117:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828356.990134:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.990153:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828356.990159:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.990168:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828356.990173:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828356.990180:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828356.990192:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236096:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828356.990205:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828356.990211:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828356.990215:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.990222:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828356.990228:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828356.990234:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828356.990238:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828356.990243:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f79f8.
00000400:00000010:2.0:1787828356.990248:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce62a8.
00000100:00000200:2.0:1787828356.990254:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236096, portal 10
00000100:00000001:2.0:1787828356.990256:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828356.990259:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:2.0:1787828356.990269:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.990277:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828356.990284:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828356.990288:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6e58.
00000100:00000200:2.0:1787828356.990291:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327236096, offset 0
00000400:00000200:2.0:1787828356.990296:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828356.990301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.990306:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828356.990313:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828356.990317:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828356.990320:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828356.990323:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828356.990327:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39616:39616:255:4294967295] 192.168.203.116@tcp LPNI seq info [39616:39616:7:4294967295]
00000400:00000200:2.0:1787828356.990331:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.990339:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828356.990343:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828356.990349:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37f00.
00000800:00000200:2.0:1787828356.990354:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828356.990359:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828356.990361:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828356.990366:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828356.990369:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828356.990371:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297177099
00000100:00000001:2.0:1787828356.990375:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297177773
00000100:00000001:2.0:1787828356.990377:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297177099
00000100:00000001:2.0:1787828356.990380:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297177773
00000100:00000001:2.0:1787828356.990382:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828356.990384:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990386:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828356.990390:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.990396:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828356.990398:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.990400:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828356.990402:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828356.990404:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828356.990407:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236224:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828356.990410:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828356.990412:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828356.990413:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.990415:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828356.990418:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828356.990420:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828356.990423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828356.990425:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7630.
00000400:00000010:2.0:1787828356.990428:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828356.990431:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236224, portal 4
00000100:00000001:2.0:1787828356.990433:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828356.990435:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:2.0:1787828356.990441:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.990448:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828356.990450:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828356.990453:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6cc0.
00000100:00000200:2.0:1787828356.990455:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327236224, offset 0
00000400:00000200:2.0:1787828356.990458:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828356.990461:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.990464:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828356.990466:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828356.990469:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828356.990471:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828356.990473:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828356.990476:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39617:39617:254:4294967295] 192.168.203.116@tcp LPNI seq info [39617:39617:6:4294967295]
00000400:00000200:2.0:1787828356.990481:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.990486:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828356.990490:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828356.990493:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37700.
00000800:00000200:2.0:1787828356.990497:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828356.990500:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828356.990502:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828356.990505:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828356.990508:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828356.990509:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297177106
00000100:00000001:2.0:1787828356.990511:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297177773
00000100:00000001:2.0:1787828356.990514:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297177106
00000100:00000001:2.0:1787828356.990516:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297177773
00000100:00000001:2.0:1787828356.990518:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828356.990520:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990523:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990525:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828356.990528:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828356.990530:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787828356.990532:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828356.990534:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828356.990535:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828356.990539:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.990546:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828356.990548:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.990550:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828356.990552:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828356.990562:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828356.990565:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828356.990569:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828356.990570:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828356.990572:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828356.990574:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828356.990595:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001000.
02000000:00000001:2.0:1787828356.990596:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828356.990600:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828356.990602:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7528.
00000400:00000010:2.0:1787828356.990605:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6990.
00000100:00000200:2.0:1787828356.990608:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236352, portal 4
00000100:00000001:2.0:1787828356.990612:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828356.990614:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:2.0:1787828356.990622:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828356.990631:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828356.990635:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828356.990639:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6b28.
00000100:00000200:2.0:1787828356.990642:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327236352, offset 0
00000400:00000200:2.0:1787828356.990648:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828356.990653:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.990657:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828356.990662:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828356.990665:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828356.990668:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828356.990705:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828356.990709:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39618:39618:253:4294967295] 192.168.203.116@tcp LPNI seq info [39618:39618:5:4294967295]
00000400:00000200:2.0:1787828356.990715:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828356.990724:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828356.990731:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828356.990737:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37b00.
00000800:00000200:2.0:1787828356.990742:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828356.990748:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828356.990752:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828356.990756:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828356.990760:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828356.990763:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297177112
00000100:00000001:2.0:1787828356.990766:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297177773
00000100:00000001:2.0:1787828356.990770:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297177112
00000100:00000001:2.0:1787828356.990773:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297177773
00000100:00000001:2.0:1787828356.990776:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828356.990780:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990783:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990787:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828356.990791:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828356.990794:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787828356.990796:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828356.990798:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828356.990801:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828356.990804:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828357.692763:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297177773, at 4297177792
00000100:00000010:2.0:1787828357.692774:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828358.844896:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828358.844900:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828358.844905:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828360.061632:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828360.061638:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828360.061641:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828360.061651:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828360]  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828360.061703:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828360.061706:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828360.061711:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7528.
00000400:00000200:2.0:1787828360.061716:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6990
00000400:00000010:2.0:1787828360.061718:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6990.
00000100:00000001:2.0:1787828360.061723:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828360.061728:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828360.061738:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.061746:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828360.061748:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.061752:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.061755:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828360.061759:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.061777:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828360.061783:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.061792:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828360.061796:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.061798:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.061800:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.061803:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828360.061807:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828360.061813:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828360.061820:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828360.061823:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828360.061828:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828360.061838:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828360.061857:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828360.061862:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828360]  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828360.061871:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828360.061873:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828360.061875:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f79f8.
00000400:00000200:2.0:1787828360.061879:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce62a8
00000400:00000010:2.0:1787828360.061880:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce62a8.
00000100:00000001:2.0:1787828360.061883:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828360.061886:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828360.061895:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.061903:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828360.061904:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.061906:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.061908:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828360.061911:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.061919:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828360.061923:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.061938:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828360.061940:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.061941:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.061943:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.061945:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828360.061948:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828360.061951:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828360.061955:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828360.061958:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828360.061963:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828360.061971:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236096:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828360.061976:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828360.061980:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828360]  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828360.061989:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828360.061990:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828360.061992:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7630.
00000400:00000200:2.0:1787828360.061995:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828360.061997:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828360.062000:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828360.062004:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828360.062013:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062020:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828360.062022:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062024:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.062025:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828360.062029:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062035:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828360.062039:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062046:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828360.062048:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062050:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.062051:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062053:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828360.062055:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828360.062057:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828360.062061:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828360.062063:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828360.062067:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828360.062074:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236224:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828360.062079:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828360.062083:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828360]  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828360.062090:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828360.062092:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828360.062094:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828360.062097:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828360.062099:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828360.062102:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828360.062105:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828360.062113:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062119:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828360.062121:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062123:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.062125:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828360.062128:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062136:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828360.062139:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062146:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828360.062149:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062150:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828360.062152:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062153:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828360.062155:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828360.062158:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828360.062162:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828360.062164:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828360.062169:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828360.062177:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327235968:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828360.062182:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062185:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828360.062189:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327236352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062198:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828360.062203:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828360.062208:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001000.
02000000:00000001:2.0:1787828360.062213:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828360.062217:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828360.062220:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828360.062223:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828360.062226:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:2.0:1787828360.062231:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828360.062234:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828360.062237:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:2.0:1787828360.062244:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828360.062246:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062247:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828360.062251:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327236096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062258:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828360.062265:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828360.062266:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828360.062269:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828360.062271:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828360.062272:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828360.062275:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828360.062277:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:2.0:1787828360.062281:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828360.062282:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828360.062284:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:2.0:1787828360.062288:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828360.062289:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062292:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828360.062296:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327236224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062304:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828360.062306:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828360.062307:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828360.062310:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828360.062312:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828360.062313:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828360.062315:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828360.062317:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:2.0:1787828360.062319:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828360.062320:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828360.062321:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787828360.062324:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828360.062325:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062327:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828360.062330:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327235968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828360.062337:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828360.062339:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828360.062341:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828360.062344:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828360.062345:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828360.062346:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828360.062348:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828360.062350:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:2.0:1787828360.062353:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828360.062354:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828360.062356:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:2.0:1787828360.062358:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828360.062360:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062364:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828360.062375:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828360.062378:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828360.062381:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828360.062384:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828361.916912:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828361.916916:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828361.916920:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828362.109371:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828362.109387:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828362.109394:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828362.109420:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.109471:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828362.109477:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.109496:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828362.109507:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828362.109520:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828362.109535:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327236480:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828362.109573:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828362.109597:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828362.109606:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.109614:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828362.109641:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828362.109650:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828362.109734:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828362.109740:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828362.109749:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828362.109756:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236480, portal 25
00000100:00000001:2.0:1787828362.109759:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828362.109761:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00000040:2.0:1787828362.109773:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.109786:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828362.109792:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828362.109798:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828362.109805:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327236480, offset 0
00000400:00000200:2.0:1787828362.109823:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828362.109867:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.109888:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828362.109906:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828362.109920:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828362.109931:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828362.109945:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828362.109960:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39619:39619:256:4294967295] 192.168.203.116@tcp LPNI seq info [39619:39619:8:4294967295]
00000400:00000200:2.0:1787828362.109974:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.110014:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828362.110026:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.110045:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828362.110067:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828362.110077:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828362.110095:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828362.110107:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828362.110129:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.110146:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828362.110164:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828362.110183:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828362.110200:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828362.110209:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828362.110222:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828362.110238:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110286:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.110299:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110336:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.110344:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828362.110358:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828362.110377:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828362.110386:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297182893
00000100:00000010:2.0:1787828362.110399:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828362.110414:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828362.110429:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297182893
00000100:00000010:2.0:1787828362.110446:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9240.
00000100:00000001:2.0:1787828362.110470:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828362.110481:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828362.110487:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.110492:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828362.110504:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828362/real 1787828362]  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828362.110518:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828362.110522:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828362.110526:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828362.110531:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828362.110533:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828362.110537:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828362.110542:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828362.110554:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110563:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828362.110565:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.110568:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.110571:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828362.110577:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110586:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828362.110592:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110603:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828362.110607:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.110610:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.110612:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.110615:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828362.110619:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828362.110624:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828362.110631:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828362.110634:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828362.110642:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828362.110708:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327236480:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828362.110718:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828362.110726:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110737:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828362.110740:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.110744:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828362.110749:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828362.110751:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828362.110755:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236608:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828362.110759:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828362.110773:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828362.110775:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.110777:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828362.110782:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828362.110785:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828362.110789:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828362.110792:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828362.110796:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828362.110800:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236608, portal 10
00000100:00000001:2.0:1787828362.110802:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828362.110804:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000100:00000040:2.0:1787828362.110811:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110825:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828362.110828:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828362.110851:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828362.110854:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327236608, offset 0
00000400:00000200:2.0:1787828362.110859:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828362.110881:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.110886:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828362.110890:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828362.110893:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828362.110896:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828362.110898:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828362.110901:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39620:39620:256:4294967295] 192.168.203.116@tcp LPNI seq info [39620:39620:8:4294967295]
00000400:00000200:2.0:1787828362.110905:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.110912:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828362.110916:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.110924:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828362.110928:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828362.110931:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828362.110935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828362.110938:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828362.110941:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.110943:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828362.110950:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828362.110953:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828362.110956:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828362.110958:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828362.110961:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828362.110964:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110983:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.110986:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.110998:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.110999:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828362.111002:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828362.111006:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828362.111009:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828362.111012:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297182220
00000100:00000001:2.0:1787828362.111014:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297182893
00000100:00000001:2.0:1787828362.111017:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297182220
00000100:00000001:2.0:1787828362.111019:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297182893
00000100:00000001:2.0:1787828362.111021:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828362.111022:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.111024:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828362.111028:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828362/real 1787828362]  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828362.111038:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828362.111040:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828362.111043:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828362.111046:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828362.111048:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828362.111051:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828362.111056:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828362.111079:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111089:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828362.111091:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111095:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.111097:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828362.111102:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111112:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828362.111117:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111126:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828362.111130:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111132:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.111134:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111137:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828362.111140:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828362.111144:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828362.111150:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828362.111153:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828362.111159:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828362.111167:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236608:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828362.111171:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111184:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.111187:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327236480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111193:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828362.111196:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828362.111198:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828362.111202:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828362.111205:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828362.111206:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828362.111209:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828362.111211:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:2.0:1787828362.111215:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828362.111217:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828362.111219:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:2.0:1787828362.111223:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828362.111224:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111226:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.111228:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327236608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111234:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828362.111235:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828362.111237:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828362.111239:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828362.111239:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828362.111241:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828362.111250:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828362.111251:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:2.0:1787828362.111253:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828362.111254:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828362.111255:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:2.0:1787828362.111258:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828362.111259:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111261:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111267:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828362.111269:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.111271:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828362.111273:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828362.111274:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828362.111278:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111284:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828362.111285:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.111288:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828362.111289:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828362.111291:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828362.111294:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236736:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828362.111297:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828362.111300:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828362.111301:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.111303:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828362.111306:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828362.111308:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828362.111311:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828362.111314:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828362.111317:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828362.111320:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236736, portal 4
00000100:00000001:2.0:1787828362.111322:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828362.111323:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:2.0:1787828362.111329:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111335:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828362.111337:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828362.111339:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828362.111341:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327236736, offset 0
00000400:00000200:2.0:1787828362.111345:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828362.111348:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.111351:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828362.111354:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828362.111357:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828362.111359:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828362.111361:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828362.111376:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39621:39621:256:4294967295] 192.168.203.116@tcp LPNI seq info [39621:39621:8:4294967295]
00000400:00000200:2.0:1787828362.111380:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.111386:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828362.111390:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.111393:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828362.111397:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828362.111399:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828362.111403:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828362.111406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828362.111409:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.111412:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828362.111414:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828362.111418:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828362.111420:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828362.111422:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828362.111424:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828362.111427:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111444:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.111447:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111453:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.111454:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828362.111457:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828362.111459:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828362.111462:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828362.111464:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297182227
00000100:00000001:2.0:1787828362.111466:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297182893
00000100:00000001:2.0:1787828362.111468:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297182227
00000100:00000001:2.0:1787828362.111470:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297182893
00000100:00000001:2.0:1787828362.111472:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828362.111474:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.111475:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828362.111478:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828362/real 1787828362]  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828362.111485:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828362.111486:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828362.111487:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828362.111489:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828362.111491:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828362.111493:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828362.111506:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828362.111513:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111519:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828362.111520:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111521:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.111522:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828362.111525:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111531:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828362.111534:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111540:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828362.111542:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111543:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.111544:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111546:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828362.111547:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828362.111550:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828362.111553:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828362.111555:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828362.111558:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828362.111565:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236736:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828362.111568:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828362.111571:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111588:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828362.111590:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.111592:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828362.111594:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828362.111595:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828362.111598:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828362.111601:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828362.111604:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828362.111606:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828362.111608:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828362.111612:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828362.111615:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828362.111619:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828362.111623:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828362.111627:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828362.111630:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236864, portal 4
00000100:00000001:2.0:1787828362.111633:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828362.111635:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:2.0:1787828362.111644:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111677:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828362.111680:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828362.111683:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828362.111686:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327236864, offset 0
00000400:00000200:2.0:1787828362.111690:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828362.111694:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.111697:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828362.111700:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828362.111703:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828362.111705:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828362.111707:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828362.111710:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39622:39622:256:4294967295] 192.168.203.116@tcp LPNI seq info [39622:39622:8:4294967295]
00000400:00000200:2.0:1787828362.111714:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828362.111720:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828362.111725:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.111727:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828362.111731:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828362.111734:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787828362.111738:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828362.111740:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787828362.111743:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828362.111745:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828362.111748:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828362.111752:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828362.111755:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828362.111756:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828362.111758:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828362.111762:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111770:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.111772:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111778:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.111780:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828362.111783:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828362.111785:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828362.111788:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828362.111790:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297182233
00000100:00000001:2.0:1787828362.111793:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297182893
00000100:00000001:2.0:1787828362.111794:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297182233
00000100:00000001:2.0:1787828362.111797:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297182893
00000100:00000001:2.0:1787828362.111799:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828362.111801:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.111803:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828362.111806:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828362/real 1787828362]  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828362.111813:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828362.111814:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828362.111816:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828362.111817:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828362.111819:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828362.111821:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828362.111823:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828362.111856:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111862:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828362.111864:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111866:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.111868:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828362.111871:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111887:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828362.111890:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111896:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828362.111898:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111899:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828362.111900:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111902:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828362.111904:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828362.111906:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828362.111909:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828362.111912:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828362.111915:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828362.111922:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327236864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828362.111925:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111927:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.111930:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327236736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111952:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828362.111954:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828362.111956:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828362.111958:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828362.111960:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828362.111961:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828362.111963:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828362.111977:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:2.0:1787828362.111979:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828362.111981:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828362.111982:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:2.0:1787828362.111984:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828362.111986:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.111988:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828362.111991:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327236864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828362.111997:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828362.111998:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828362.112000:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828362.112002:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828362.112003:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828362.112004:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828362.112005:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828362.112007:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:2.0:1787828362.112009:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828362.112010:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828362.112012:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787828362.112014:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828362.112015:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.112017:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828362.112020:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828362.112022:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828362.112024:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828362.112025:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828362.812856:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297182893, at 4297182912
00000100:00000010:2.0:1787828362.812874:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828364.989030:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828364.989037:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828364.989041:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828367.229338:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828367.229355:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828367.229361:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828367.229390:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.229439:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828367.229451:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.229477:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828367.229485:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828367.229500:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828367.229532:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327236992:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828367.229573:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828367.229586:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828367.229592:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.229613:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828367.229630:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828367.229643:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828367.229755:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828367.229773:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828367.229824:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828367.229842:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327236992, portal 25
00000100:00000001:2.0:1787828367.229854:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828367.229861:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:2.0:1787828367.229887:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.229943:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828367.229959:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828367.229969:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828367.229984:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327236992, offset 0
00000400:00000200:2.0:1787828367.230008:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828367.230028:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.230043:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828367.230057:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828367.230078:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828367.230097:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828367.230108:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828367.230128:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39623:39623:256:4294967295] 192.168.203.116@tcp LPNI seq info [39623:39623:8:4294967295]
00000400:00000200:2.0:1787828367.230155:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.230180:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828367.230198:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.230209:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828367.230226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828367.230241:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787828367.230265:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828367.230287:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787828367.230316:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.230333:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828367.230352:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828367.230381:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828367.230401:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828367.230410:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828367.230426:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828367.230446:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.230515:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.230535:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.230610:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.230622:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828367.230644:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828367.230736:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828367.230749:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297188014
00000100:00000010:2.0:1787828367.230764:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828367.230781:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828367.230821:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297188014
00000100:00000010:2.0:1787828367.230832:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee96c0.
00000100:00000001:2.0:1787828367.230848:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828367.230871:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828367.230878:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.230902:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828367.230922:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828367/real 1787828367]  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828367.230972:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828367.230984:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828367.230989:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828367.230998:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828367.231006:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828367.231016:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828367.231022:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828367.231046:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231065:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828367.231072:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231088:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.231094:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828367.231117:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231152:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828367.231172:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231221:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828367.231228:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231232:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.231237:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231243:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828367.231251:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828367.231271:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828367.231281:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828367.231289:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828367.231299:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828367.231318:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327236992:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828367.231328:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231332:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.231340:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327236992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231354:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828367.231359:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828367.231363:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828367.231368:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828367.231371:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828367.231375:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828367.231378:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828367.231381:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:2.0:1787828367.231386:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828367.231389:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828367.231391:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:2.0:1787828367.231396:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828367.231398:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231401:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231410:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828367.231412:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.231416:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828367.231418:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828367.231421:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828367.231427:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231437:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828367.231439:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.231442:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828367.231446:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828367.231449:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828367.231453:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237120:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828367.231457:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828367.231461:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828367.231463:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.231465:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828367.231470:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828367.231472:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828367.231478:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828367.231482:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828367.231486:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828367.231491:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237120, portal 10
00000100:00000001:2.0:1787828367.231494:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828367.231496:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:2.0:1787828367.231504:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231514:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828367.231518:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828367.231523:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828367.231528:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327237120, offset 0
00000400:00000200:2.0:1787828367.231537:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828367.231546:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.231555:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828367.231561:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828367.231566:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828367.231572:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828367.231576:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828367.231584:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39624:39624:256:4294967295] 192.168.203.116@tcp LPNI seq info [39624:39624:8:4294967295]
00000400:00000200:2.0:1787828367.231593:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.231607:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828367.231617:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.231623:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828367.231629:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828367.231633:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787828367.231640:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828367.231643:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787828367.231649:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.231651:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828367.231689:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828367.231694:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828367.231698:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828367.231701:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828367.231704:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828367.231710:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231721:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.231725:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231743:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.231745:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828367.231750:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828367.231755:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828367.231760:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828367.231763:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297187341
00000100:00000001:2.0:1787828367.231767:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297188014
00000100:00000001:2.0:1787828367.231770:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297187341
00000100:00000001:2.0:1787828367.231773:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297188014
00000100:00000001:2.0:1787828367.231775:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828367.231778:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.231781:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828367.231785:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828367/real 1787828367]  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828367.231818:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828367.231820:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828367.231823:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828367.231826:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828367.231828:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828367.231831:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828367.231836:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828367.231847:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231855:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828367.231857:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231859:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.231861:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828367.231865:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231874:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828367.231879:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231887:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828367.231890:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231892:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.231894:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.231896:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828367.231899:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828367.231902:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828367.231906:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828367.231909:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828367.231914:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828367.231923:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237120:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828367.231928:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828367.231933:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.231942:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828367.231944:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.231947:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828367.231950:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828367.231953:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828367.231956:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237248:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828367.231960:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828367.231963:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828367.231966:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.231968:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828367.231973:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828367.231975:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828367.231981:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828367.231984:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828367.231988:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828367.231992:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237248, portal 4
00000100:00000001:2.0:1787828367.231994:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828367.231996:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:2.0:1787828367.232004:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232012:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828367.232015:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828367.232018:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828367.232021:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327237248, offset 0
00000400:00000200:2.0:1787828367.232026:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828367.232031:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.232035:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828367.232039:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828367.232043:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828367.232046:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828367.232049:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828367.232053:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39625:39625:256:4294967295] 192.168.203.116@tcp LPNI seq info [39625:39625:8:4294967295]
00000400:00000200:2.0:1787828367.232058:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.232067:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828367.232072:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.232076:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828367.232081:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828367.232084:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787828367.232090:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828367.232094:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787828367.232099:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.232102:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828367.232106:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828367.232111:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828367.232114:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828367.232117:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828367.232119:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828367.232124:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232134:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.232138:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232147:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.232149:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828367.232153:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828367.232157:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828367.232160:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828367.232163:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297187348
00000100:00000001:2.0:1787828367.232166:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297188014
00000100:00000001:2.0:1787828367.232169:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297187348
00000100:00000001:2.0:1787828367.232172:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297188014
00000100:00000001:2.0:1787828367.232174:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828367.232177:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.232179:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828367.232183:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828367/real 1787828367]  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828367.232192:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828367.232194:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828367.232196:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828367.232199:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828367.232201:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828367.232204:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828367.232208:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828367.232218:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232226:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828367.232227:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232229:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.232231:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828367.232235:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232243:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828367.232248:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232256:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828367.232261:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232264:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.232266:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232270:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828367.232272:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828367.232277:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828367.232283:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828367.232288:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828367.232296:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828367.232312:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237248:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828367.232320:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828367.232329:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232345:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828367.232348:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.232353:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828367.232356:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828367.232361:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828367.232367:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828367.232372:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828367.232375:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828367.232377:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828367.232380:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828367.232385:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001000.
02000000:00000001:2.0:1787828367.232387:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828367.232391:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828367.232395:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828367.232399:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828367.232403:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237376, portal 4
00000100:00000001:2.0:1787828367.232406:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828367.232408:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:2.0:1787828367.232416:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232424:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828367.232427:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828367.232430:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828367.232433:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327237376, offset 0
00000400:00000200:2.0:1787828367.232437:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828367.232442:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.232447:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828367.232451:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828367.232454:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828367.232457:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828367.232460:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828367.232465:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39626:39626:256:4294967295] 192.168.203.116@tcp LPNI seq info [39626:39626:8:4294967295]
00000400:00000200:2.0:1787828367.232470:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828367.232479:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828367.232485:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.232489:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828367.232494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828367.232497:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787828367.232503:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828367.232506:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787828367.232536:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828367.232540:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828367.232546:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828367.232554:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828367.232561:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828367.232565:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828367.232571:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828367.232579:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232597:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.232604:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828392 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232619:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.232621:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828367.232627:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828367.232631:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828367.232636:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828367.232639:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297187353
00000100:00000001:2.0:1787828367.232642:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297188014
00000100:00000001:2.0:1787828367.232645:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297187353
00000100:00000001:2.0:1787828367.232648:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297188014
00000100:00000001:2.0:1787828367.232651:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828367.232683:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.232686:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828367.232692:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828367/real 1787828367]  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828367.232703:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828367.232705:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828367.232707:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828367.232711:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828367.232713:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828367.232716:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828367.232719:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828367.232730:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232738:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828367.232740:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232742:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.232744:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828367.232748:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232756:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828367.232761:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232769:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828367.232772:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232774:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828367.232775:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232778:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828367.232780:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828367.232783:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828367.232788:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828367.232791:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828367.232819:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828367.232832:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828367.232837:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232840:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.232844:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327237120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232852:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828367.232855:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828367.232858:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828367.232861:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828367.232863:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828367.232865:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828367.232868:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828367.232870:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:2.0:1787828367.232874:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828367.232876:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828367.232878:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:2.0:1787828367.232881:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828367.232883:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232885:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.232889:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327237248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232896:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828367.232898:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828367.232900:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828367.232903:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828367.232904:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828367.232906:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828367.232908:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828367.232910:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:2.0:1787828367.232913:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828367.232914:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828367.232916:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:2.0:1787828367.232919:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828367.232921:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232923:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828367.232926:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327237376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828392 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828367.232934:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828367.232935:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828367.232937:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001000.
02000000:00000001:2.0:1787828367.232939:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828367.232941:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828367.232942:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828367.232944:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828367.232946:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:2.0:1787828367.232948:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828367.232950:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828367.232951:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:2.0:1787828367.232954:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828367.232955:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232957:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828367.232962:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828367.232964:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828367.232966:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828367.232969:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828367.932866:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297188014, at 4297188032
00000100:00000010:2.0:1787828367.932888:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828368.060904:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828368.060912:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828368.060918:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828371.132934:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828371.132940:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828371.132945:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828372.349128:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828372.349136:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828372.349140:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828372.349149:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349173:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828372.349177:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.349184:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828372.349186:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828372.349191:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828372.349198:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327237504:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828372.349204:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828372.349210:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828372.349215:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.349221:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828372.349228:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828372.349233:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828372.349241:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828372.349247:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828372.349255:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828372.349264:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237504, portal 25
00000100:00000001:2.0:1787828372.349270:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828372.349273:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418994624 : -116696290556992 : ffff95dd870e03c0)
00000100:00000040:2.0:1787828372.349282:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349297:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828372.349304:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828372.349318:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828372.349323:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327237504, offset 0
00000400:00000200:2.0:1787828372.349336:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828372.349350:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.349361:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828372.349370:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828372.349379:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828372.349387:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828372.349393:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828372.349408:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39627:39627:256:4294967295] 192.168.203.116@tcp LPNI seq info [39627:39627:8:4294967295]
00000400:00000200:2.0:1787828372.349417:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.349435:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828372.349445:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.349452:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828372.349465:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828372.349473:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787828372.349485:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828372.349491:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:2.0:1787828372.349499:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.349502:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828372.349512:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828372.349523:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828372.349532:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828372.349536:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828372.349544:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828372.349551:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349575:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.349582:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349610:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.349628:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828372.349639:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828372.349652:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828372.349733:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297193133
00000100:00000010:2.0:1787828372.349744:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828372.349778:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828372.349815:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297193133
00000100:00000010:2.0:1787828372.349824:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9180.
00000100:00000001:2.0:1787828372.349832:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828372.349848:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828372.349855:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.349861:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828372.349881:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828372/real 1787828372]  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828372.349900:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828372.349902:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828372.349905:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828372.349908:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828372.349910:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828372.349913:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828372.349916:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828372.349926:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349934:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828372.349936:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.349938:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.349941:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828372.349946:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349954:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828372.349959:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.349964:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828372.349968:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.349969:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.349970:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.349972:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828372.349974:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828372.349976:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828372.349979:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828372.349981:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828372.349984:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828372.349990:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327237504:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828372.349994:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.349996:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.349998:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e03c0 x1874671327237504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350002:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828372.350004:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350006:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828372.350008:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828372.350010:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828372.350022:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828372.350024:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828372.350026:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:2.0:1787828372.350029:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828372.350031:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828372.350032:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e03c0.
00000100:00000001:2.0:1787828372.350034:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828372.350035:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350037:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350044:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828372.350045:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350047:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828372.350048:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828372.350049:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828372.350052:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350062:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828372.350063:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.350064:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828372.350066:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828372.350067:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828372.350069:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237632:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828372.350071:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828372.350072:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828372.350074:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.350075:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350077:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828372.350078:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828372.350081:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828372.350083:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828372.350086:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828372.350088:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237632, portal 10
00000100:00000001:2.0:1787828372.350090:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828372.350091:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:2.0:1787828372.350097:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350101:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828372.350103:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828372.350105:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828372.350106:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327237632, offset 0
00000400:00000200:2.0:1787828372.350109:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828372.350112:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.350114:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828372.350117:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828372.350119:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828372.350120:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828372.350122:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828372.350125:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39628:39628:256:4294967295] 192.168.203.116@tcp LPNI seq info [39628:39628:8:4294967295]
00000400:00000200:2.0:1787828372.350128:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.350133:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828372.350136:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.350139:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828372.350142:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828372.350144:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787828372.350147:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828372.350149:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:2.0:1787828372.350151:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.350153:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828372.350155:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828372.350158:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828372.350160:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828372.350161:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828372.350163:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828372.350165:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350171:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.350173:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350178:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350179:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828372.350181:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828372.350185:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828372.350187:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828372.350188:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297192459
00000100:00000001:2.0:1787828372.350190:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297193133
00000100:00000001:2.0:1787828372.350192:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297192459
00000100:00000001:2.0:1787828372.350194:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297193133
00000100:00000001:2.0:1787828372.350195:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828372.350197:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350198:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828372.350200:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828372/real 1787828372]  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828372.350206:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828372.350207:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828372.350208:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828372.350209:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828372.350210:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828372.350212:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828372.350214:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828372.350219:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350223:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828372.350224:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350225:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.350226:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828372.350228:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350232:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828372.350235:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350239:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828372.350240:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350242:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.350243:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350244:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828372.350245:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828372.350248:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828372.350250:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828372.350252:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828372.350255:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828372.350260:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237632:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828372.350262:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828372.350264:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350268:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828372.350269:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.350270:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828372.350271:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828372.350272:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828372.350274:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828372.350276:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828372.350277:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828372.350278:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.350279:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350281:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828372.350283:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828372.350285:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828372.350287:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828372.350289:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828372.350291:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237760, portal 4
00000100:00000001:2.0:1787828372.350293:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828372.350294:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:2.0:1787828372.350297:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350302:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828372.350304:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828372.350305:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828372.350307:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327237760, offset 0
00000400:00000200:2.0:1787828372.350320:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828372.350322:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.350324:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828372.350327:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828372.350328:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828372.350330:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828372.350332:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828372.350334:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39629:39629:256:4294967295] 192.168.203.116@tcp LPNI seq info [39629:39629:8:4294967295]
00000400:00000200:2.0:1787828372.350337:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.350341:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828372.350344:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.350346:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828372.350349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828372.350351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787828372.350354:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828372.350355:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:2.0:1787828372.350358:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.350359:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828372.350361:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828372.350363:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828372.350365:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828372.350367:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828372.350368:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828372.350371:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350375:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.350377:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350381:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350383:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828372.350385:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828372.350398:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828372.350400:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828372.350402:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297192466
00000100:00000001:2.0:1787828372.350403:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297193133
00000100:00000001:2.0:1787828372.350405:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297192466
00000100:00000001:2.0:1787828372.350407:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297193133
00000100:00000001:2.0:1787828372.350408:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828372.350409:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350410:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828372.350412:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828372/real 1787828372]  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828372.350417:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828372.350418:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828372.350418:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828372.350420:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828372.350421:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828372.350423:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828372.350425:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828372.350430:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350434:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828372.350435:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350436:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.350437:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828372.350439:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350443:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828372.350445:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350450:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828372.350452:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350453:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.350454:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350455:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828372.350456:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828372.350457:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828372.350459:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828372.350460:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828372.350463:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828372.350467:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828372.350470:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828372.350472:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350475:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828372.350476:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.350477:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828372.350478:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828372.350480:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828372.350481:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237888:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828372.350483:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828372.350484:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828372.350485:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828372.350486:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350489:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001000.
02000000:00000001:2.0:1787828372.350490:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828372.350492:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828372.350495:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828372.350497:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828372.350499:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327237888, portal 4
00000100:00000001:2.0:1787828372.350500:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828372.350501:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:2.0:1787828372.350505:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350509:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828372.350510:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828372.350512:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828372.350513:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327237888, offset 0
00000400:00000200:2.0:1787828372.350516:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828372.350518:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.350521:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828372.350522:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828372.350524:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828372.350526:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828372.350528:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828372.350530:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39630:39630:256:4294967295] 192.168.203.116@tcp LPNI seq info [39630:39630:8:4294967295]
00000400:00000200:2.0:1787828372.350533:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828372.350537:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828372.350540:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.350541:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828372.350543:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828372.350545:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787828372.350548:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828372.350549:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:2.0:1787828372.350563:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828372.350564:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828372.350566:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828372.350569:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828372.350570:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828372.350571:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828372.350574:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828372.350576:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350581:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.350583:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350587:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350589:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828372.350590:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828372.350592:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828372.350594:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828372.350595:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297192471
00000100:00000001:2.0:1787828372.350597:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297193133
00000100:00000001:2.0:1787828372.350599:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297192471
00000100:00000001:2.0:1787828372.350600:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297193133
00000100:00000001:2.0:1787828372.350601:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828372.350635:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350637:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828372.350640:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828372/real 1787828372]  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828372.350646:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828372.350647:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828372.350649:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828372.350650:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828372.350652:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828372.350654:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828372.350657:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828372.350665:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350715:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828372.350718:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350721:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.350723:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828372.350729:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350740:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828372.350746:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350771:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828372.350785:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350787:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828372.350790:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350793:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828372.350795:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828372.350798:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828372.350818:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828372.350820:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828372.350823:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828372.350828:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327237888:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828372.350831:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350832:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.350834:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327237632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350850:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828372.350851:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350854:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828372.350855:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828372.350857:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828372.350858:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828372.350859:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828372.350860:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:2.0:1787828372.350862:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828372.350863:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828372.350864:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:2.0:1787828372.350866:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828372.350868:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350869:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.350871:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327237760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350875:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828372.350876:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350877:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828372.350879:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828372.350880:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828372.350880:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828372.350881:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828372.350882:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:2.0:1787828372.350884:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828372.350884:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828372.350885:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:2.0:1787828372.350886:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828372.350887:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350888:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828372.350890:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327237888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828397 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828372.350893:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828372.350894:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828372.350895:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001000.
02000000:00000001:2.0:1787828372.350896:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828372.350897:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828372.350897:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828372.350899:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828372.350900:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:2.0:1787828372.350902:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828372.350903:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828372.350904:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:2.0:1787828372.350905:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828372.350906:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350907:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828372.350909:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828372.350910:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828372.350911:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828372.350912:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828373.052809:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297193133, at 4297193152
00000100:00000010:2.0:1787828373.052827:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828373.180982:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828373.180994:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828373.181003:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828374.204909:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828374.204917:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828374.204923:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828376.252954:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828376.252959:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828376.252963:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828377.276980:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828377.276985:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828377.277000:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828377.469060:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828377.469066:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828377.469070:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828377.469079:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469091:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828377.469094:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.469098:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828377.469102:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828377.469105:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828377.469109:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327238016:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828377.469115:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828377.469121:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828377.469124:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.469128:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828377.469134:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828377.469136:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828377.469144:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828377.469149:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828377.469158:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828377.469167:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238016, portal 25
00000100:00000001:2.0:1787828377.469170:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828377.469172:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:2.0:1787828377.469179:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469201:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828377.469221:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828377.469224:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828377.469227:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327238016, offset 0
00000400:00000200:2.0:1787828377.469235:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828377.469245:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.469251:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828377.469259:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828377.469264:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828377.469285:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828377.469289:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828377.469295:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39631:39631:256:4294967295] 192.168.203.116@tcp LPNI seq info [39631:39631:8:4294967295]
00000400:00000200:2.0:1787828377.469312:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.469327:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828377.469335:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.469353:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828377.469363:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828377.469368:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828377.469376:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828377.469381:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787828377.469387:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.469391:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828377.469399:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828377.469405:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828377.469412:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828377.469416:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828377.469422:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828377.469427:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469439:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.469445:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469456:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.469458:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828377.469465:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828377.469470:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828377.469474:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297198252
00000100:00000010:2.0:1787828377.469480:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828377.469489:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828377.469497:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297198252
00000100:00000010:2.0:1787828377.469500:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9ea0.
00000100:00000001:2.0:1787828377.469504:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828377.469509:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828377.469513:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.469516:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828377.469521:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377]  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828377.469534:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828377.469537:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828377.469540:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828377.469543:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828377.469545:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828377.469548:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828377.469552:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828377.469563:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469572:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828377.469575:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469578:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.469581:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828377.469587:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469607:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828377.469611:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469617:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828377.469620:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469621:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.469623:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469625:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828377.469626:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828377.469629:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828377.469632:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828377.469634:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828377.469637:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828377.469644:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327238016:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828377.469649:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469651:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.469678:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327238016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469685:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828377.469688:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828377.469690:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828377.469694:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828377.469696:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828377.469697:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828377.469699:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828377.469702:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:2.0:1787828377.469706:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828377.469708:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828377.469710:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:2.0:1787828377.469713:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828377.469714:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469716:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469741:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828377.469744:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.469758:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828377.469759:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828377.469761:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828377.469765:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469771:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828377.469772:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.469774:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828377.469776:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828377.469777:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828377.469780:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238144:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828377.469783:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828377.469784:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828377.469786:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.469788:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828377.469791:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828377.469792:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828377.469796:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828377.469799:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828377.469802:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828377.469805:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238144, portal 10
00000100:00000001:2.0:1787828377.469806:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828377.469808:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:2.0:1787828377.469814:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469820:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828377.469822:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828377.469823:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828377.469825:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327238144, offset 0
00000400:00000200:2.0:1787828377.469828:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828377.469832:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.469835:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828377.469838:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828377.469840:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828377.469842:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828377.469844:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828377.469847:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39632:39632:256:4294967295] 192.168.203.116@tcp LPNI seq info [39632:39632:8:4294967295]
00000400:00000200:2.0:1787828377.469850:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.469856:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828377.469859:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.469862:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828377.469866:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828377.469868:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828377.469872:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828377.469874:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787828377.469877:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.469879:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828377.469882:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828377.469885:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828377.469888:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828377.469889:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828377.469891:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828377.469894:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469901:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.469903:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469908:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.469910:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828377.469912:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828377.469916:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828377.469918:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828377.469920:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297197579
00000100:00000001:2.0:1787828377.469922:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297198252
00000100:00000001:2.0:1787828377.469924:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297197579
00000100:00000001:2.0:1787828377.469926:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297198252
00000100:00000001:2.0:1787828377.469928:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828377.469929:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.469931:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828377.469934:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377]  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828377.469939:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828377.469940:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828377.469941:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828377.469943:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828377.469944:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828377.469946:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828377.469949:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828377.469972:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469976:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828377.469977:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469978:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.469979:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828377.469981:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469985:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828377.469987:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.469991:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828377.469993:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469994:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.469994:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.469996:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828377.469997:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828377.469999:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828377.470001:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828377.470003:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828377.470006:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828377.470010:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238144:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828377.470012:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828377.470014:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470040:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828377.470042:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.470044:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828377.470045:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828377.470047:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828377.470049:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828377.470051:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828377.470052:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828377.470053:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.470055:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828377.470057:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828377.470059:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828377.470062:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828377.470063:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828377.470066:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828377.470068:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238272, portal 4
00000100:00000001:2.0:1787828377.470070:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828377.470071:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:2.0:1787828377.470076:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470081:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828377.470082:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828377.470084:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828377.470086:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327238272, offset 0
00000400:00000200:2.0:1787828377.470089:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828377.470092:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.470094:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828377.470096:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828377.470098:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828377.470100:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828377.470102:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828377.470105:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39633:39633:256:4294967295] 192.168.203.116@tcp LPNI seq info [39633:39633:8:4294967295]
00000400:00000200:2.0:1787828377.470109:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.470114:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828377.470118:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.470120:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828377.470123:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828377.470125:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828377.470128:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828377.470130:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787828377.470133:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.470135:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828377.470137:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828377.470140:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828377.470142:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828377.470144:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828377.470146:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828377.470148:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470154:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.470156:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470161:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.470163:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828377.470165:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828377.470167:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828377.470170:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828377.470172:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297197586
00000100:00000001:2.0:1787828377.470174:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297198252
00000100:00000001:2.0:1787828377.470175:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297197586
00000100:00000001:2.0:1787828377.470177:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297198252
00000100:00000001:2.0:1787828377.470178:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828377.470180:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.470182:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828377.470185:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377]  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828377.470191:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828377.470192:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828377.470193:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828377.470194:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828377.470212:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828377.470213:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828377.470215:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828377.470220:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470225:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828377.470226:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470227:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.470228:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828377.470230:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470234:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828377.470236:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470240:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828377.470241:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470242:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.470243:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470244:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828377.470245:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828377.470246:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828377.470249:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828377.470250:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828377.470253:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828377.470258:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828377.470260:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828377.470279:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470284:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828377.470285:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.470286:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828377.470287:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828377.470289:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828377.470291:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238400:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828377.470293:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828377.470294:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828377.470295:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828377.470297:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828377.470299:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828377.470300:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828377.470303:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828377.470304:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828377.470306:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828377.470308:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238400, portal 4
00000100:00000001:2.0:1787828377.470310:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828377.470311:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:2.0:1787828377.470316:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470321:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828377.470323:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828377.470325:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828377.470327:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327238400, offset 0
00000400:00000200:2.0:1787828377.470330:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828377.470332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.470335:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828377.470337:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828377.470339:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828377.470341:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828377.470343:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828377.470345:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39634:39634:256:4294967295] 192.168.203.116@tcp LPNI seq info [39634:39634:8:4294967295]
00000400:00000200:2.0:1787828377.470348:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828377.470354:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828377.470357:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.470359:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828377.470361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828377.470363:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787828377.470366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828377.470368:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787828377.470371:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828377.470373:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828377.470375:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828377.470377:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828377.470379:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828377.470381:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828377.470383:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828377.470385:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470390:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.470392:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470398:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.470399:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828377.470402:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828377.470404:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828377.470406:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828377.470408:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297197591
00000100:00000001:2.0:1787828377.470410:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297198252
00000100:00000001:2.0:1787828377.470412:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297197591
00000100:00000001:2.0:1787828377.470414:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297198252
00000100:00000001:2.0:1787828377.470415:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828377.470416:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.470418:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828377.470420:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377]  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828377.470426:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828377.470427:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828377.470428:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828377.470430:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828377.470431:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828377.470433:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828377.470435:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828377.470441:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470445:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828377.470446:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470447:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.470448:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828377.470451:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470456:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828377.470458:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470464:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828377.470466:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470467:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828377.470469:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470470:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828377.470471:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828377.470473:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828377.470475:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828377.470477:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828377.470479:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828377.470485:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238400:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828377.470487:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470489:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.470491:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327238144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470495:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828377.470497:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828377.470498:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828377.470500:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828377.470501:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828377.470502:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828377.470503:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828377.470504:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:2.0:1787828377.470506:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828377.470508:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828377.470509:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:2.0:1787828377.470511:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828377.470512:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470514:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.470517:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327238272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470522:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828377.470523:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828377.470524:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828377.470536:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828377.470537:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828377.470537:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828377.470539:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828377.470540:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:2.0:1787828377.470541:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828377.470542:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828377.470543:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:2.0:1787828377.470544:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828377.470545:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470546:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828377.470566:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327238400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828377.470572:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828377.470573:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828377.470575:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828377.470577:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828377.470578:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828377.470579:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828377.470581:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828377.470583:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:2.0:1787828377.470584:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828377.470586:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828377.470587:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:2.0:1787828377.470589:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828377.470590:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470592:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828377.470595:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828377.470597:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828377.470599:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828377.470602:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828378.172715:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297198252, at 4297198272
00000100:00000010:2.0:1787828378.172749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828379.324988:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828379.324995:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828379.325001:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828380.348923:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828380.348930:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828380.348936:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828382.396884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828382.396890:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828382.396894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828382.589267:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828382.589285:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828382.589291:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828382.589321:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.589363:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828382.589371:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.589380:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828382.589388:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828382.589401:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828382.589416:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327238528:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828382.589441:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828382.589453:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828382.589461:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.589467:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828382.589482:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828382.589488:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828382.589508:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828382.589517:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828382.589546:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828382.589569:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238528, portal 25
00000100:00000001:2.0:1787828382.589581:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828382.589592:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418994624 : -116696290556992 : ffff95dd870e03c0)
00000100:00000040:2.0:1787828382.589615:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.589647:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828382.589751:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828382.589755:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828382.589760:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327238528, offset 0
00000400:00000200:2.0:1787828382.589770:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828382.589799:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.589815:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828382.589825:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828382.589833:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828382.589841:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828382.589847:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828382.589862:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39635:39635:256:4294967295] 192.168.203.116@tcp LPNI seq info [39635:39635:8:4294967295]
00000400:00000200:2.0:1787828382.589875:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.589908:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828382.589924:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.589935:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828382.589958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828382.589974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828382.590022:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828382.590032:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787828382.590060:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.590071:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828382.590082:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828382.590097:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828382.590138:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828382.590147:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828382.590157:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828382.590168:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590210:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.590225:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590257:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.590273:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828382.590289:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828382.590305:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828382.590314:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297203373
00000100:00000010:2.0:1787828382.590334:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9660.
00000100:00000001:2.0:1787828382.590356:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828382.590372:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297203373
00000100:00000010:2.0:1787828382.590381:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9ba0.
00000100:00000001:2.0:1787828382.590401:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787828382.590412:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828382.590434:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.590459:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828382.590484:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382]  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828382.590515:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828382.590521:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828382.590538:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828382.590546:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828382.590556:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828382.590567:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828382.590579:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828382.590616:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590655:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828382.590662:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.590672:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.590681:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828382.590751:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590798:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828382.590814:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590831:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828382.590836:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.590840:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.590841:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.590845:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828382.590848:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828382.590852:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828382.590859:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828382.590862:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828382.590865:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828382.590872:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327238528:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828382.590877:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.590879:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.590882:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e03c0 x1874671327238528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590888:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828382.590891:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828382.590894:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828382.590898:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828382.590900:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828382.590902:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787828382.590904:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828382.590907:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:2.0:1787828382.590911:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828382.590915:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828382.590917:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e03c0.
00000100:00000001:2.0:1787828382.590920:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828382.590921:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.590923:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.590932:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828382.590934:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.590936:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828382.590938:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828382.590939:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828382.590943:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.590950:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828382.590952:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.590965:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828382.590967:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828382.590969:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828382.590971:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238656:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828382.590973:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828382.590975:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828382.590977:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.590978:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828382.590981:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828382.590983:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828382.590986:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828382.590989:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828382.590993:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828382.590996:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238656, portal 10
00000100:00000001:2.0:1787828382.590998:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828382.591015:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:2.0:1787828382.591023:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591029:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828382.591032:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828382.591034:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828382.591037:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327238656, offset 0
00000400:00000200:2.0:1787828382.591040:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828382.591045:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.591048:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828382.591052:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828382.591054:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828382.591057:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828382.591059:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828382.591063:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39636:39636:256:4294967295] 192.168.203.116@tcp LPNI seq info [39636:39636:8:4294967295]
00000400:00000200:2.0:1787828382.591066:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.591073:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828382.591076:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.591079:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828382.591082:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828382.591085:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828382.591090:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828382.591092:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787828382.591096:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.591098:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828382.591101:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828382.591114:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828382.591117:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828382.591119:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828382.591121:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828382.591124:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591131:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.591134:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591140:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591142:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828382.591145:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828382.591150:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787828382.591152:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828382.591155:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297202700
00000100:00000001:2.0:1787828382.591157:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297203373
00000100:00000001:2.0:1787828382.591160:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297202700
00000100:00000001:2.0:1787828382.591162:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297203373
00000100:00000001:2.0:1787828382.591164:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828382.591166:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591168:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828382.591171:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382]  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828382.591177:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828382.591178:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828382.591186:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828382.591188:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828382.591189:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828382.591191:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828382.591194:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828382.591200:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591206:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828382.591208:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591209:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.591211:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828382.591213:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591219:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828382.591222:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591229:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828382.591231:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591233:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.591234:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591236:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828382.591238:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828382.591240:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828382.591242:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828382.591244:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828382.591247:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828382.591253:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238656:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828382.591256:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828382.591260:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591265:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828382.591266:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.591268:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828382.591270:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828382.591271:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828382.591273:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828382.591276:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828382.591289:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828382.591290:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.591292:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828382.591295:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828382.591297:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828382.591301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828382.591303:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828382.591306:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828382.591309:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238784, portal 4
00000100:00000001:2.0:1787828382.591311:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828382.591312:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:2.0:1787828382.591317:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591321:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828382.591323:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828382.591325:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828382.591328:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327238784, offset 0
00000400:00000200:2.0:1787828382.591331:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828382.591334:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.591336:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828382.591339:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828382.591341:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828382.591343:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828382.591345:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828382.591348:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39637:39637:256:4294967295] 192.168.203.116@tcp LPNI seq info [39637:39637:8:4294967295]
00000400:00000200:2.0:1787828382.591352:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.591357:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828382.591360:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.591363:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828382.591365:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828382.591368:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828382.591372:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828382.591374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787828382.591378:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.591380:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828382.591382:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828382.591385:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828382.591387:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828382.591388:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828382.591391:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828382.591394:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591401:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.591403:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591408:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591410:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828382.591412:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828382.591414:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787828382.591416:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828382.591418:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297202707
00000100:00000001:2.0:1787828382.591420:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297203373
00000100:00000001:2.0:1787828382.591423:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297202707
00000100:00000001:2.0:1787828382.591425:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297203373
00000100:00000001:2.0:1787828382.591426:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828382.591428:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591429:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828382.591432:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382]  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828382.591438:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828382.591439:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828382.591440:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828382.591442:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828382.591443:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828382.591445:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828382.591448:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828382.591455:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591466:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828382.591468:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591469:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.591471:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828382.591474:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591479:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828382.591482:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591487:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828382.591489:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591490:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.591491:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591493:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828382.591494:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828382.591496:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828382.591498:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828382.591500:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828382.591503:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828382.591509:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828382.591512:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787828382.591515:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591521:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828382.591522:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.591525:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828382.591526:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828382.591528:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828382.591531:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238912:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828382.591534:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787828382.591535:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828382.591537:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828382.591538:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787828382.591541:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828382.591542:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787828382.591545:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787828382.591547:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787828382.591550:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787828382.591552:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327238912, portal 4
00000100:00000001:2.0:1787828382.591554:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828382.591567:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:2.0:1787828382.591572:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591578:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787828382.591580:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787828382.591582:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787828382.591584:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327238912, offset 0
00000400:00000200:2.0:1787828382.591587:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828382.591589:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.591592:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787828382.591594:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787828382.591596:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787828382.591598:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787828382.591600:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787828382.591603:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39638:39638:256:4294967295] 192.168.203.116@tcp LPNI seq info [39638:39638:8:4294967295]
00000400:00000200:2.0:1787828382.591607:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828382.591612:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787828382.591617:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.591619:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787828382.591633:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787828382.591635:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787828382.591639:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787828382.591641:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787828382.591664:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787828382.591666:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787828382.591669:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828382.591672:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828382.591676:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787828382.591677:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787828382.591679:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787828382.591683:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591743:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.591746:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591752:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591753:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787828382.591757:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828382.591760:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787828382.591763:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828382.591765:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297202713
00000100:00000001:2.0:1787828382.591767:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297203373
00000100:00000001:2.0:1787828382.591769:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297202713
00000100:00000001:2.0:1787828382.591771:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297203373
00000100:00000001:2.0:1787828382.591774:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828382.591776:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591777:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787828382.591780:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382]  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787828382.591788:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787828382.591790:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787828382.591791:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787828382.591793:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787828382.591795:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787828382.591796:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787828382.591799:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787828382.591806:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591811:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828382.591812:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591814:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.591815:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787828382.591818:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591823:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828382.591826:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591831:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787828382.591833:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591834:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787828382.591835:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591837:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787828382.591838:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828382.591840:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828382.591843:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787828382.591844:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787828382.591848:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828382.591855:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327238912:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787828382.591858:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591859:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.591862:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327238656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591867:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828382.591868:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828382.591870:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003400.
02000000:00000001:2.0:1787828382.591871:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828382.591873:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828382.591874:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787828382.591876:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828382.591877:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:2.0:1787828382.591880:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828382.591881:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828382.591882:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:2.0:1787828382.591885:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828382.591886:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591887:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.591890:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327238784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591895:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828382.591896:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828382.591897:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000800.
02000000:00000001:2.0:1787828382.591899:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828382.591900:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828382.591901:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787828382.591902:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828382.591903:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:2.0:1787828382.591905:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828382.591906:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828382.591907:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:2.0:1787828382.591909:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828382.591911:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591912:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828382.591915:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327238912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828382.591919:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828382.591920:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787828382.591921:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787828382.591923:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828382.591924:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787828382.591925:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787828382.591926:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787828382.591927:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:2.0:1787828382.591928:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787828382.591929:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787828382.591930:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:2.0:1787828382.591932:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828382.591933:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591934:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828382.591937:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828382.591938:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828382.591940:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828382.591945:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828383.292844:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297203373, at 4297203392
00000100:00000010:2.0:1787828383.292860:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9660.
00000020:00000001:2.0:1787828383.420931:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828383.420936:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828383.420952:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828385.468818:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828385.468821:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828385.468824:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828386.492940:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828386.492949:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828386.492979:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828387.708933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828387.708956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828387.708959:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828387.708964:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828387.708970:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828387.708979:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2535
00000400:00000200:2.0:1787828387.708990:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828387.708998:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828387.709003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828387.709007:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709011:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.709014:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709017:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828387.709022:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828387.709025:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828387.709028:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828387.709032:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828387.709035:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828387.709038:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828387.709042:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709045:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.709047:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709050:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828387.709054:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828387.709058:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709062:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828387.709065:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828387.709069:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828387.709073:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709076:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828387.709080:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828387.709102:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828387.709107:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828387.709112:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828387.709114:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709117:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828387.709121:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828387.709126:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828387.709136:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828387.709143:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828387.709156:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327239040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828387.709169:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828387.709175:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828387.709181:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327239040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828387.709217:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828387.709221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828387.709226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828387.709228:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828387.709230:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828387.709233:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828387.709237:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2535
00000400:00000200:2.0:1787828387.709242:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828387.709246:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828387.709249:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828387.709254:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709258:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.709261:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709264:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828387.709268:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828387.709271:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828387.709273:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828387.709277:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828387.709288:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828387.709293:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828387.709303:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709308:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.709314:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709320:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828387.709326:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828387.709339:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709346:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828387.709352:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828387.709365:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828387.709373:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709377:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828387.709385:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828387.709397:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709405:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828387.709418:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828387.709428:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828387.709448:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828387.709452:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709466:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828387.709471:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828387.709482:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828387.709493:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828387.709511:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828387.709525:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327239168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828387.709560:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828387.709563:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828387.709576:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327239168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828387.709597:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828387.709601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828387.709607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828387.709621:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828387.709636:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828387.709729:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828387.709739:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2535
00000400:00000200:2.0:1787828387.709753:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828387.709762:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828387.709766:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828387.709779:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709784:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.709789:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709792:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828387.709803:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828387.709807:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828387.709811:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828387.709814:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828387.709824:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828387.709835:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828387.709843:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709848:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.709861:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709870:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828387.709887:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828387.709902:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828387.709912:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828387.709917:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828387.709927:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828387.709947:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709953:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828387.709962:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828387.709967:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.709969:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828387.709975:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828387.709982:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828387.709989:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828387.709997:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.710008:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828387.710013:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828387.710024:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828387.710035:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828387.710052:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828387.710067:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327239296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828387.710100:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828387.710103:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828387.710115:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327239296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828387.710147:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828387.710148:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787828387.710160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.1:1787828387.710161:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00080000:2.0:1787828387.710181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828387.710189:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828387.710202:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828387.710221:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828387.710228:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2535
00000400:00000200:2.0:1787828387.710246:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828387.710262:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828387.710267:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828387.710281:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.710291:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.710297:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.710306:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828387.710315:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828387.710322:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828387.710330:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828387.710333:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828387.710343:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828387.710352:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828387.710362:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.710365:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828387.710368:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828387.710376:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828387.710386:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828387.710395:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828387.710407:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828387.710420:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828387.710427:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828387.710432:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.710439:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828387.710444:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828387.710448:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.710453:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828387.710459:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828387.710469:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828387.710475:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828387.710486:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828387.710490:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828387.710494:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828387.710501:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828387.710508:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828387.710521:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828387.710537:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327239424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828387.710567:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828387.710580:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828387.710594:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327239424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828387.710625:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828387.710635:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2545)
00000800:00000200:3.1:1787828387.711113:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828387.711115:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828387.711409:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828387.711411:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828387.711767:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828387.711769:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787828388.412785:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297208493, at 4297208512
00000100:00000010:2.0:1787828388.412802:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3ff60.
00000020:00000001:2.0:1787828388.540949:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828388.540957:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828388.540961:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828389.564924:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828389.564932:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828389.564938:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828391.612959:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828391.612967:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828391.612974:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828392.636909:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828392.636916:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828392.636921:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828392.830115:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828392.830126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828392.831046:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828392.831049:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828392.831469:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828392.831474:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828392.831994:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828392.831998:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828393.532787:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297213613, at 4297213632
00000100:00000010:2.0:1787828393.532810:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f5a0.
00000020:00000001:2.0:1787828394.684998:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828394.685004:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828394.685009:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828395.708972:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828395.708976:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828395.708980:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828397.756864:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828397.756868:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828397.756872:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828397.949657:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828397.949660:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828397.950404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828397.950406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828397.950686:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828397.950689:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828397.951064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828397.951067:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828398.652779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297218733, at 4297218752
00000100:00000010:2.0:1787828398.652805:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f6c0.
00000020:00000001:2.0:1787828398.780872:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828398.780879:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828398.780885:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828400.828924:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828400.828932:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828400.828939:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828401.852843:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828401.852849:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828401.852855:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828403.069611:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828403.069621:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828403.070704:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828403.070708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828403.071121:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828403.071125:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828403.071770:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828403.071773:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828403.772791:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297223853, at 4297223872
00000100:00000010:2.0:1787828403.772815:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f240.
00000020:00000001:2.0:1787828403.900853:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828403.900857:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828403.900860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828404.924928:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828404.924935:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828404.924939:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828406.652853:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828406.652859:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828406.652861:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828406.652867:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828406.652871:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828406.652875:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828406.652894:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828406.652897:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828406.652902:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828406.972839:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828406.972843:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828406.972847:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828407.676824:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828407.676833:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828407.676836:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828407.676844:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828407.676849:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828407.676854:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828407.676858:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828407.676861:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828407.676865:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828407.996878:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828407.996883:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828407.996888:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828408.189750:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828408.189757:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828408.190358:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828408.190360:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828408.190589:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828408.190591:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828408.190807:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828408.190809:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787828408.700758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828408.700778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828408.700782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828408.700788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828408.700816:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828408.700821:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828408.700825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828408.700828:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828408.700832:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828408.892853:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297228973, at 4297228992
00000100:00000010:2.0:1787828408.892907:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f9c0.
00000100:00000001:2.0:1787828409.724754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828409.724765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828409.724768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828409.724775:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828409.724780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828409.724784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828409.724788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828409.724791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828409.724795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828410.044872:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828410.044878:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828410.044882:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828410.748782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828410.748791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828410.748793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828410.748799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828410.748804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828410.748808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828410.748812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828410.748816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828410.748819:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828411.068862:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828411.068865:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828411.068868:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828411.772797:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828411.772807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828411.772810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828411.772816:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828411.772821:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828411.772826:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828411.772829:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828411.772832:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828411.772836:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828412.796808:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828412.796819:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828412.796822:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828412.796830:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828412.796835:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828412.796840:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828412.796844:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828412.796847:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828412.796851:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828413.116960:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828413.116966:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828413.116971:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828413.310023:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828413.310032:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787828413.310794:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828413.310797:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787828413.311198:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828413.311201:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787828413.311578:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828413.311581:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000100:00000001:2.0:1787828413.820736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828413.820745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828413.820747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828413.820753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828413.820757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828413.820762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828413.820765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828413.820767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828413.820771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828414.012794:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297234093, at 4297234112
00000100:00000010:2.0:1787828414.012818:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f420.
00000020:00000001:2.0:1787828414.140972:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828414.140977:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828414.140982:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828414.844801:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828414.844812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828414.844816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828414.844824:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828414.844829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828414.844835:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828414.844838:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828414.844842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828414.844846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828415.868721:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828415.868730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828415.868732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828415.868739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828415.868742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828415.868753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828415.868765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828415.868767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828415.868770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828416.188889:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828416.188894:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828416.188898:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828416.892791:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828416.892801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828416.892805:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828416.892812:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828416.892817:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828416.892822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828416.892826:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828416.892830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828416.892833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828417.212898:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828417.212902:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828417.212906:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828417.916784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828417.916792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828417.916795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828417.916801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828417.916805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828417.916809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828417.916812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828417.916815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828417.916818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828418.429983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828418.430007:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828418.430978:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828418.430981:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828418.431426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828418.431429:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828418.431944:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828418.431948:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1787828418.940760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828418.940768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828418.940771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828418.940777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828418.940782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828418.940786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828418.940790:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828418.940793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828418.940795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828419.132788:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297239213, at 4297239232
00000100:00000010:2.0:1787828419.132816:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f600.
00000020:00000001:2.0:1787828419.260923:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828419.260931:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828419.260937:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828419.964809:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828419.964818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828419.964821:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828419.964827:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828419.964832:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828419.964836:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828419.964840:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828419.964843:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828419.964847:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828420.284948:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828420.284954:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828420.284959:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828420.988829:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828420.988840:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828420.988844:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828420.988852:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828420.988858:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828420.988864:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828420.988868:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828420.988873:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828420.988878:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828422.012814:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828422.012827:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828422.012831:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828422.012856:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828422.012862:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828422.012866:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828422.012870:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828422.012874:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828422.012877:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828422.332889:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828422.332895:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828422.332900:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828423.036730:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828423.036739:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828423.036741:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828423.036747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828423.036750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828423.036754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828423.036757:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828423.036759:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828423.036762:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828423.356823:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828423.356828:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828423.356833:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828423.549332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828423.549814:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828423.549993:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828423.550159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828424.060869:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828424.060879:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828424.060882:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828424.060888:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828424.060893:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828424.060897:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828424.060900:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828424.060904:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828424.060907:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828424.252727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297244332, at 4297244352
00000100:00000010:2.0:1787828424.252736:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000001:2.0:1787828425.084745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828425.084757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828425.084760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828425.084768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828425.084774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828425.084780:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828425.084784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828425.084789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828425.084793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828425.404942:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828425.404952:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828425.404959:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828426.108849:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828426.108860:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828426.108863:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828426.108870:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828426.108875:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828426.108881:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828426.108885:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828426.108889:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828426.108893:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828426.428931:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828426.428937:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828426.428941:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828427.132766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828427.132777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828427.132779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828427.132786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828427.132791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828427.132795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828427.132816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828427.132820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828427.132825:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828428.156813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828428.156825:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828428.156830:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828428.156839:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828428.156845:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828428.156850:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828428.156854:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828428.156858:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828428.156863:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828428.476957:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828428.476966:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828428.476973:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828428.669974:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828428.669979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828428.670895:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828428.670898:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828428.671232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828428.671234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828428.671558:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828428.671562:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787828429.180818:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828429.180831:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828429.180834:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828429.180842:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828429.180847:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828429.180853:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828429.180857:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828429.180860:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828429.180864:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828429.372727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297249453, at 4297249472
00000100:00000010:2.0:1787828429.372736:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000020:00000001:2.0:1787828429.500922:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828429.500926:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828429.500929:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828430.204771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828430.204784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828430.204788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828430.204797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828430.204802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828430.204806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828430.204809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828430.204812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828430.204815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828431.228755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828431.228768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828431.228772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828431.228781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828431.228789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828431.228795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828431.228800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828431.228805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828431.228810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828431.548879:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828431.548884:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828431.548887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828432.252732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828432.252739:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828432.252741:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828432.252745:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828432.252749:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828432.252752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828432.252765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828432.252768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828432.252771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828432.572871:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828432.572875:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828432.572879:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828433.276805:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828433.276814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828433.276818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828433.276824:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828433.276829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828433.276851:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828433.276857:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828433.276861:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828433.276865:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828433.789982:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828433.789988:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828433.791003:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828433.791006:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828433.791410:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828433.791413:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828433.791860:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828433.791863:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787828434.300823:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828434.300834:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828434.300837:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828434.300844:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828434.300850:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828434.300855:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828434.300859:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828434.300862:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828434.300866:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828434.492849:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297254573, at 4297254592
00000100:00000010:2.0:1787828434.492872:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000020:00000001:2.0:1787828434.620911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828434.620915:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828434.620918:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828435.644950:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828435.644956:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828435.644961:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828437.692945:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828437.692951:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828437.692956:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828438.716935:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828438.716944:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828438.716952:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828438.908869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828438.908890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828438.908897:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828438.908905:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828438.908915:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828438.908930:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2586
00000400:00000200:2.0:1787828438.908949:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828438.908963:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828438.908970:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828438.908978:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.908984:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.908989:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.908995:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828438.909004:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828438.909009:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828438.909015:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828438.909022:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828438.909028:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828438.909031:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828438.909037:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909040:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.909043:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909046:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828438.909050:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828438.909055:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909058:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828438.909063:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828438.909068:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828438.909072:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909075:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828438.909079:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828438.909095:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828438.909100:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828438.909106:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828438.909109:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909114:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828438.909118:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828438.909126:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828438.909139:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828438.909150:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828438.909164:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327244160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828438.909181:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828438.909187:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828438.909194:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327244160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828438.909229:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828438.909234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828438.909239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828438.909241:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828438.909245:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828438.909249:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828438.909252:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2586
00000400:00000200:2.0:1787828438.909258:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828438.909262:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828438.909266:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828438.909269:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909273:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.909275:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909278:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828438.909280:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828438.909282:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828438.909284:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828438.909288:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828438.909291:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828438.909294:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828438.909322:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909325:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.909329:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909332:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828438.909336:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828438.909341:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909344:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828438.909347:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828438.909357:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828438.909366:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909371:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828438.909383:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828438.909391:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909395:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828438.909403:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828438.909412:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828438.909421:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828438.909426:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909433:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828438.909439:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828438.909444:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828438.909455:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828438.909464:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828438.909479:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327244288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828438.909498:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828438.909505:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828438.909514:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327244288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828438.909536:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828438.909546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828438.909554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828438.909561:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828438.909564:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828438.909568:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828438.909581:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2586
00000400:00000200:2.0:1787828438.909595:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828438.909603:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828438.909606:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828438.909615:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909629:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.909639:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909649:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828438.909728:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828438.909734:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828438.909738:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828438.909742:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828438.909754:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828438.909762:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828438.909772:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909779:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.909783:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909786:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828438.909793:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828438.909802:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828438.909811:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828438.909818:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828438.909827:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828438.909836:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909846:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828438.909856:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828438.909866:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909871:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828438.909886:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828438.909896:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828438.909903:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828438.909908:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.909915:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828438.909923:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828438.909928:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828438.909938:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828438.909951:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828438.909972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327244416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828438.910004:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828438.910015:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828438.910034:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327244416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828438.910078:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828438.910085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828438.910100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828438.910106:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828438.910113:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828438.910123:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828438.910132:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2586
00000400:00000200:2.0:1787828438.910145:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828438.910153:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828438.910159:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828438.910169:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.910176:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.910184:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.910189:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828438.910201:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828438.910204:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828438.910206:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828438.910210:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828438.910212:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828438.910215:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828438.910221:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828438.910227:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828438.910230:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787828438.910234:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828438.910235:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828438.910245:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000200:0.1:1787828438.910248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00080000:2.0:1787828438.910253:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828438.910263:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828438.910271:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828438.910279:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828438.910292:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.910321:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828438.910329:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828438.910340:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.910348:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828438.910355:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828438.910363:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828438.910371:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828438.910374:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828438.910380:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828438.910383:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828438.910390:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828438.910403:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828438.910413:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828438.910428:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327244544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828438.910464:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828438.910473:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828438.910486:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327244544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828438.910504:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828438.910513:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2596)
00000800:00000200:0.1:1787828438.911161:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828438.911165:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787828438.911724:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828438.911730:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787828438.912237:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828438.912243:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000040:2.0:1787828439.612780:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297259693, at 4297259712
00000100:00000010:2.0:1787828439.612795:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000020:00000001:2.0:1787828440.764992:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828440.765000:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828440.765007:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000008:00000020:2.0:1787828440.956778:0:16289:0:(osc_request.c:747:osc_update_next_shrink()) next time 3793 to shrink grant
00000008:00000020:2.0:1787828440.956794:0:16289:0:(osc_request.c:747:osc_update_next_shrink()) next time 3793 to shrink grant
00000020:00000001:2.0:1787828441.788909:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828441.788917:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828441.788924:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828443.836870:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828443.836874:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828443.836878:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828444.029635:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828444.029695:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787828444.030353:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828444.030356:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787828444.030712:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828444.030714:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787828444.031008:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828444.031010:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1787828444.732729:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297264813, at 4297264832
00000100:00000010:2.0:1787828444.732747:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000020:00000001:2.0:1787828444.860854:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828444.860858:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828444.860861:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828446.908870:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828446.908875:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828446.908878:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828447.932827:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828447.932832:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828447.932835:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828449.149762:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828449.149779:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787828449.150869:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828449.150871:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787828449.151160:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828449.151163:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787828449.151498:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828449.151501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1787828449.852780:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297269933, at 4297269952
00000100:00000010:2.0:1787828449.852799:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000020:00000001:2.0:1787828449.980969:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828449.980975:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828449.980979:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828451.004918:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828451.004925:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828451.004932:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828453.052918:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828453.052924:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828453.052929:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828454.077009:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828454.077019:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828454.077026:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828454.270097:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828454.270118:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828454.270952:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828454.270954:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828454.271232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828454.271234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828454.271474:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828454.271476:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787828454.972809:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297275053, at 4297275072
00000100:00000010:2.0:1787828454.972837:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000020:00000001:2.0:1787828456.124999:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828456.125008:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828456.125016:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828457.148965:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828457.148972:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828457.148977:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828459.196886:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828459.196890:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828459.196894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828459.388893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828459.388908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828459.388912:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828459.388919:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828459.388927:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828459.388939:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2607
00000400:00000200:2.0:1787828459.388952:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828459.388963:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828459.388968:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828459.388973:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.388977:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.388997:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389000:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828459.389006:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828459.389008:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828459.389012:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828459.389016:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828459.389018:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828459.389021:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828459.389024:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389026:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.389028:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389030:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828459.389034:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828459.389038:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389043:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828459.389047:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828459.389052:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828459.389055:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389058:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828459.389061:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828459.389074:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828459.389078:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828459.389083:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828459.389102:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389107:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828459.389112:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828459.389119:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828459.389132:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828459.389142:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828459.389192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327246208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828459.389222:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828459.389227:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828459.389232:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327246208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828459.389268:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828459.389271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828459.389275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828459.389277:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828459.389279:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828459.389282:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828459.389285:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2607
00000400:00000200:2.0:1787828459.389288:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828459.389292:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828459.389295:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828459.389299:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389303:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.389306:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389309:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828459.389311:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828459.389313:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828459.389315:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828459.389318:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828459.389323:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828459.389326:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828459.389330:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389335:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.389340:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389343:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828459.389348:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828459.389357:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389362:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828459.389367:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828459.389376:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828459.389384:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389394:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828459.389401:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828459.389407:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389411:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828459.389421:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828459.389431:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828459.389441:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828459.389446:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389454:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828459.389458:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828459.389462:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828459.389477:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828459.389489:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828459.389504:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327246336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828459.389517:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828459.389521:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828459.389529:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327246336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828459.389552:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828459.389557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828459.389561:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828459.389563:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828459.389565:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828459.389572:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828459.389581:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2607
00000400:00000200:2.0:1787828459.389592:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828459.389600:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828459.389606:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828459.389611:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389619:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.389627:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389630:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828459.389632:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828459.389633:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828459.389638:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828459.389644:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828459.389649:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828459.389728:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828459.389742:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389752:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.389763:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389776:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828459.389788:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828459.389806:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828459.389810:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828459.389816:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828459.389822:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828459.389830:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389837:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828459.389844:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828459.389851:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389855:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828459.389863:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828459.389870:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828459.389879:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828459.389884:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.389890:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828459.389894:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828459.389897:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828459.389906:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828459.389919:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828459.389931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327246464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787828459.389958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828459.389960:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828459.389964:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828459.389971:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327246464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828459.389997:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828459.390000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787828459.390007:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00080000:2.0:1787828459.390008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828459.390013:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828459.390017:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828459.390026:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828459.390033:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2607
00000400:00000200:2.0:1787828459.390040:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828459.390047:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828459.390051:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828459.390061:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.390065:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.390070:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.390076:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828459.390080:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828459.390082:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828459.390084:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828459.390086:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828459.390108:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828459.390112:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828459.390121:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.390128:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828459.390134:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828459.390141:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828459.390150:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828459.390181:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828459.390185:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828459.390192:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828459.390215:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828459.390217:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.390220:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828459.390225:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828459.390229:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.390233:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828459.390238:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828459.390256:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828459.390263:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828459.390266:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828459.390270:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828459.390276:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828459.390278:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828459.390287:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828459.390294:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828459.390302:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327246592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828459.390333:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828459.390338:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828459.390348:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327246592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828459.390359:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828459.390367:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2617)
00000800:00000200:0.1:1787828459.390744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828459.390746:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828459.391026:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828459.391029:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828459.391326:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828459.391328:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828460.092788:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297280173, at 4297280192
00000100:00000010:2.0:1787828460.092814:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000020:00000001:2.0:1787828460.221063:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828460.221070:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828460.221075:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828462.268852:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828462.268857:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828462.268861:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828463.292881:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828463.292887:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828463.292891:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828464.509950:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828464.509963:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828464.510986:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828464.510988:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828464.511390:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828464.511393:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828464.511749:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828464.511752:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828465.212782:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297285293, at 4297285312
00000100:00000010:2.0:1787828465.212806:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000020:00000001:2.0:1787828465.340856:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828465.340862:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828465.340866:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828466.364840:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828466.364846:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828466.364850:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828468.412861:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828468.412866:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828468.412870:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828469.436867:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828469.436874:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828469.436879:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828469.629989:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828469.629997:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828469.630852:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828469.630855:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828469.631317:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828469.631320:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828469.631808:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828469.631811:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828470.332764:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297290413, at 4297290432
00000100:00000010:2.0:1787828470.332783:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000020:00000001:2.0:1787828471.484959:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828471.484966:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828471.484971:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828472.509029:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828472.509034:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828472.509038:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828474.557066:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828474.557090:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828474.557094:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828474.749983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828474.750020:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828474.750935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828474.750939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828474.751464:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828474.751468:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828474.751961:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828474.751964:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828475.452745:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297295533, at 4297295552
00000100:00000010:2.0:1787828475.452761:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000020:00000001:2.0:1787828475.580847:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828475.580853:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828475.580857:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828477.628938:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828477.628944:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828477.628949:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828478.652959:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828478.652965:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828478.652970:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828479.869673:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828479.869677:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828479.870500:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828479.870502:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828479.870838:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828479.870840:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828479.871084:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828479.871087:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1787828480.572765:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297300653, at 4297300672
00000100:00000010:2.0:1787828480.572780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000020:00000001:2.0:1787828480.701032:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828480.701038:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828480.701042:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828481.724913:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828481.724919:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828481.724924:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828483.772875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828483.772882:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828483.772886:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828484.796918:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828484.796926:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828484.796932:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828484.989428:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828484.989435:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828484.989898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828484.989900:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828484.990120:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828484.990122:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828484.990306:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828484.990308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000040:2.0:1787828485.692809:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297305772, at 4297305792
00000100:00000010:2.0:1787828485.692835:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fcc0.
00000020:00000001:2.0:1787828486.844868:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828486.844873:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828486.844876:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828487.868994:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828487.868999:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828487.869003:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828489.916905:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828489.916910:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828489.916914:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828490.110068:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828490.110766:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828490.111065:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828490.111329:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828490.812778:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297310893, at 4297310912
00000100:00000010:2.0:1787828490.812796:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000020:00000001:2.0:1787828490.940914:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828490.940919:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828490.940923:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828492.989081:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828492.989088:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828492.989092:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828494.013033:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828494.013041:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828494.013048:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828495.230019:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828495.230026:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787828495.231087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828495.231091:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787828495.231486:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828495.231488:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1787828495.231895:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828495.231898:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 57 secs later)
00000100:00000040:2.0:1787828495.932791:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297316013, at 4297316032
00000100:00000010:2.0:1787828495.932811:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000020:00000001:2.0:1787828496.061021:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828496.061031:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828496.061039:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828497.084917:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828497.084925:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828497.084931:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828499.132919:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828499.132926:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828499.132932:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828500.157068:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828500.157074:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828500.157080:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828500.350023:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828500.350035:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787828500.351040:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828500.351043:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787828500.351452:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828500.351456:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787828500.351899:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828500.351913:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000100:00000040:2.0:1787828501.052779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297321133, at 4297321152
00000100:00000010:2.0:1787828501.052815:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3ff60.
00000020:00000001:2.0:1787828502.204924:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828502.204932:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828502.204940:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828503.228948:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828503.228955:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828503.228961:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828505.276941:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828505.276957:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828505.276961:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828505.469918:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828505.469924:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828505.470778:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828505.470781:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828505.471230:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828505.471233:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828505.471633:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828505.471637:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00000040:2.0:1787828506.172754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297326253, at 4297326272
00000100:00000010:2.0:1787828506.172766:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f5a0.
00000020:00000001:2.0:1787828506.300869:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828506.300875:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828506.300878:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828508.348977:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828508.348982:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828508.348987:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828509.373042:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828509.373050:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828509.373056:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828510.588817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828510.588846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828510.588850:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828510.588855:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828510.588862:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828510.588871:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2658
00000400:00000200:2.0:1787828510.588882:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828510.588890:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828510.588894:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828510.588898:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.588901:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.588904:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.588906:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828510.588911:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828510.588913:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828510.588916:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828510.588920:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828510.588922:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828510.588924:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828510.588926:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.588928:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.588930:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.588932:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828510.588935:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828510.588938:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828510.588941:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828510.588944:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828510.588948:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828510.588950:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.588953:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828510.588955:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828510.588968:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787828510.588972:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828510.588989:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828510.588991:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.588994:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828510.588998:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828510.589003:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828510.589012:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828510.589020:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828510.589031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327251328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828510.589053:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828510.589058:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828510.589063:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327251328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828510.589088:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828510.589092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828510.589095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828510.589096:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828510.589098:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828510.589101:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828510.589103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2658
00000400:00000200:2.0:1787828510.589105:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828510.589108:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828510.589115:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828510.589118:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589120:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.589122:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589125:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828510.589127:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828510.589129:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828510.589130:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828510.589132:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828510.589135:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828510.589137:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828510.589141:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589147:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.589149:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589151:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828510.589156:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828510.589162:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589165:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828510.589167:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828510.589186:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828510.589192:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589198:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828510.589206:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828510.589211:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589217:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828510.589227:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828510.589231:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828510.589258:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828510.589262:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589265:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828510.589266:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828510.589269:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828510.589274:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828510.589280:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828510.589290:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327251456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828510.589299:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828510.589301:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828510.589304:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327251456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828510.589331:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828510.589335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828510.589342:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828510.589346:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828510.589351:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828510.589355:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828510.589361:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2658
00000400:00000200:2.0:1787828510.589371:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828510.589378:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828510.589381:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828510.589388:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589393:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.589400:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589405:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828510.589409:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828510.589411:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828510.589415:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828510.589420:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828510.589423:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828510.589424:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828510.589428:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589433:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.589437:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589439:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828510.589441:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828510.589448:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589452:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828510.589457:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828510.589471:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828510.589474:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589478:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828510.589482:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828510.589487:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589493:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828510.589501:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828510.589507:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828510.589512:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828510.589513:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589515:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828510.589519:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828510.589523:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828510.589539:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828510.589542:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828510.589548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327251584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828510.589558:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828510.589562:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828510.589569:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327251584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828510.589582:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828510.589600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828510.589607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:3.1:1787828510.589608:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828510.589609:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828510.589612:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828510.589617:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.1:1787828510.589619:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00080000:2.0:1787828510.589622:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2658
00000400:00000200:2.0:1787828510.589629:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828510.589635:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828510.589641:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828510.589645:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589652:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.589739:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589747:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828510.589754:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828510.589760:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828510.589765:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828510.589769:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828510.589776:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828510.589781:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828510.589793:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589819:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828510.589824:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589828:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828510.589832:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828510.589839:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828510.589863:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828510.589867:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828510.589872:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828510.589879:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589884:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828510.589886:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828510.589889:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589891:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828510.589899:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828510.589905:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828510.589909:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828510.589913:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828510.589915:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828510.589916:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828510.589918:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828510.589929:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828510.589937:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828510.589943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327251712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828510.589952:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828510.589957:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828510.589967:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327251712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828510.589983:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828510.589999:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2668)
00000800:00000200:3.1:1787828510.590293:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828510.590307:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828510.590582:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828510.590585:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828510.591012:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828510.591015:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000040:2.0:1787828511.292741:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297331373, at 4297331392
00000100:00000010:2.0:1787828511.292755:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f2a0.
00000020:00000001:2.0:1787828511.420964:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828511.420969:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828511.420974:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828512.444959:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828512.444966:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828512.444972:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828514.492986:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828514.492992:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828514.492997:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828515.516971:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828515.516978:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828515.516982:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828515.709750:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828515.709756:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828515.710697:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828515.710701:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828515.711216:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828515.711220:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828515.711600:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828515.711603:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1787828516.412853:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297336493, at 4297336512
00000100:00000010:2.0:1787828516.412893:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000020:00000001:2.0:1787828517.564927:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828517.564935:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828517.564941:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828518.588868:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828518.588872:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828518.588876:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828520.636986:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828520.636994:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828520.637002:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828520.829843:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828520.829852:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828520.830374:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828520.830377:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828520.830797:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828520.830800:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828520.831072:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828520.831075:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787828521.532901:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297341613, at 4297341632
00000100:00000010:2.0:1787828521.532924:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000020:00000001:2.0:1787828521.660978:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828521.660987:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828521.660994:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828523.709022:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828523.709032:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828523.709039:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828524.732986:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828524.732996:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828524.733006:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828525.949981:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828525.949996:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828525.950938:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828525.950942:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828525.951410:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828525.951414:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828525.951910:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828525.951914:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828526.652915:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297346733, at 4297346752
00000100:00000010:2.0:1787828526.652928:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000020:00000001:2.0:1787828526.780912:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828526.780934:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828526.780938:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828527.804872:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828527.804885:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828527.804889:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828529.852893:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828529.852898:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828529.852902:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828530.876932:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828530.876939:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828530.876944:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828531.070214:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828531.070231:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828531.071551:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828531.071554:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828531.072003:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828531.072005:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828531.072347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828531.072350:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828531.772840:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297351853, at 4297351872
00000100:00000010:2.0:1787828531.772870:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000020:00000001:2.0:1787828532.924855:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828532.924859:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828532.924861:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828533.948950:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828533.948956:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828533.948962:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828535.996841:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828535.996845:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828535.996849:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828536.190030:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828536.190037:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828536.190780:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828536.190782:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828536.191099:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828536.191101:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828536.191377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828536.191379:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828536.892789:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297356973, at 4297356992
00000100:00000010:2.0:1787828536.892814:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fea0.
00000020:00000001:2.0:1787828537.020859:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828537.020865:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828537.020869:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828539.068804:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828539.068808:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828539.068811:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828540.092923:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828540.092927:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828540.092931:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828541.308818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828541.308839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828541.308844:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828541.308851:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828541.308859:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828541.308871:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2688
00000400:00000200:2.0:1787828541.308885:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828541.308895:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828541.308901:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828541.308906:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.308910:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.308914:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.308917:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828541.308923:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828541.308926:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828541.308930:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828541.308935:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828541.308939:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828541.308943:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828541.308947:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.308950:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.308953:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.308955:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828541.308959:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828541.308965:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828541.308968:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828541.308972:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828541.308977:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828541.308981:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.308984:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828541.308988:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828541.309003:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828541.309008:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828541.309014:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828541.309016:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309021:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828541.309025:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828541.309032:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828541.309044:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828541.309055:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828541.309069:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327254400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828541.309085:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828541.309090:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828541.309098:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327254400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828541.309131:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828541.309135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828541.309140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828541.309143:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828541.309146:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828541.309150:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828541.309153:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2688
00000400:00000200:2.0:1787828541.309158:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828541.309162:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828541.309165:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828541.309169:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309172:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.309176:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309178:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828541.309182:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828541.309185:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828541.309188:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828541.309191:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828541.309193:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828541.309196:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828541.309201:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309205:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.309208:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309211:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828541.309214:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828541.309220:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309225:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828541.309232:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828541.309242:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828541.309248:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309256:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828541.309264:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828541.309270:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309275:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828541.309288:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828541.309297:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828541.309305:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828541.309311:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309320:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828541.309323:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828541.309333:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828541.309348:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828541.309360:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828541.309394:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327254528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828541.309414:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828541.309422:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828541.309435:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327254528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828541.309460:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828541.309464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828541.309474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828541.309476:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828541.309479:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828541.309485:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828541.309495:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2688
00000400:00000200:2.0:1787828541.309508:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828541.309520:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828541.309530:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828541.309543:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309551:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.309561:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309571:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828541.309579:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828541.309609:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828541.309616:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828541.309623:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828541.309629:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828541.309638:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828541.309649:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309652:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.309713:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309723:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828541.309731:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828541.309744:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828541.309759:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828541.309773:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828541.309785:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828541.309797:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309809:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828541.309818:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828541.309826:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309835:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828541.309849:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828541.309859:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828541.309866:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828541.309870:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.309882:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828541.309883:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828541.309886:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828541.309895:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828541.309904:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828541.309918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327254656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787828541.309936:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828541.309944:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787828541.309946:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000100:00000001:2.0:1787828541.309952:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828541.309971:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327254656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828541.309986:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828541.309998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828541.310009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828541.310016:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828541.310021:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828541.310032:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828541.310041:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2688
00000400:00000200:2.0:1787828541.310058:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828541.310071:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828541.310081:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828541.310092:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.310098:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.310111:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.310121:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828541.310131:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828541.310136:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828541.310145:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828541.310152:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828541.310162:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828541.310166:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828541.310172:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.310175:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828541.310182:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828541.310187:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828541.310195:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828541.310203:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828541.310210:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828541.310218:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828541.310226:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828541.310229:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.310232:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828541.310239:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828541.310245:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.310247:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828541.310254:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828541.310264:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828541.310268:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828541.310270:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828541.310274:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828541.310280:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828541.310285:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828541.310293:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828541.310305:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828541.310324:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327254784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828541.310343:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828541.310352:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828541.310368:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327254784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828541.310389:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828541.310402:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2699)
00000800:00000200:3.1:1787828541.310896:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828541.310899:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787828541.311313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828541.311316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787828541.311540:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828541.311542:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000100:00000040:2.0:1787828542.012779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297362093, at 4297362112
00000100:00000010:2.0:1787828542.012797:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f0c0.
00000020:00000001:2.0:1787828542.140882:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828542.140889:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828542.140894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828542.844826:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828542.844836:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828542.844840:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828542.844848:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828542.844854:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828542.844859:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828542.844865:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828542.844869:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828542.844873:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828543.164906:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828543.164913:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828543.164918:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828543.868715:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828543.868722:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828543.868723:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828543.868727:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828543.868730:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828543.868734:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828543.868736:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828543.868739:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828543.868741:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828544.892826:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828544.892834:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828544.892837:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828544.892843:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828544.892858:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828544.892862:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828544.892865:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828544.892868:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828544.892871:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828545.213007:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828545.213015:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828545.213020:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828545.916742:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828545.916754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828545.916759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828545.916770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828545.916778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828545.916784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828545.916789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828545.916795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828545.916801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828546.236933:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828546.236937:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828546.236941:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828546.430288:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828546.430313:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787828546.431393:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828546.431397:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787828546.431937:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828546.431941:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787828546.432387:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828546.432391:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000100:00000001:2.0:1787828546.940790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828546.940798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828546.940800:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828546.940804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828546.940808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828546.940812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828546.940814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828546.940817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828546.940820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828547.132746:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297367213, at 4297367232
00000100:00000010:2.0:1787828547.132765:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000100:00000001:2.0:1787828547.964848:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828547.964863:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828547.964867:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828547.964876:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828547.964881:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828547.964886:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828547.964890:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828547.964894:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828547.964898:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828548.284963:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828548.284972:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828548.284979:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828548.988816:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828548.988847:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828548.988850:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828548.988860:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828548.988865:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828548.988872:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828548.988877:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828548.988882:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828548.988886:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828549.308851:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828549.308856:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828549.308861:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828550.012739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828550.012746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828550.012747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828550.012753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828550.012769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828550.012773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828550.012776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828550.012779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828550.012782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828551.036765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828551.036773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828551.036792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828551.036797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828551.036801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828551.036805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828551.036808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828551.036811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828551.036814:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828551.356967:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828551.356977:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828551.356982:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828551.549798:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828551.549810:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787828551.550897:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828551.550901:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787828551.551266:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828551.551271:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787828551.551688:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828551.551690:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1787828552.060828:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828552.060836:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828552.060838:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828552.060844:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828552.060848:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828552.060853:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828552.060856:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828552.060859:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828552.060862:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828552.252792:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297372333, at 4297372352
00000100:00000010:2.0:1787828552.252815:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000020:00000001:2.0:1787828552.380913:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828552.380919:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828552.380924:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828553.084775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828553.084786:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828553.084789:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828553.084796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828553.084800:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828553.084804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828553.084807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828553.084809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828553.084812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828554.108774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828554.108784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828554.108786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828554.108792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828554.108796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828554.108800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828554.108809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828554.108812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828554.108815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828554.428973:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828554.428982:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828554.428988:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828555.132780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828555.132789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828555.132792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828555.132798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828555.132803:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828555.132807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828555.132811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828555.132814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828555.132818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828555.452981:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828555.452986:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828555.452990:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828556.156796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828556.156805:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828556.156808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828556.156815:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828556.156819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828556.156824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828556.156828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828556.156831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828556.156834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828556.668812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828556.668824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828556.668827:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828556.668832:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828556.668837:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828556.668845:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2704
00000400:00000200:2.0:1787828556.668855:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828556.668863:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828556.668868:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828556.668872:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.668876:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.668879:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.668883:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828556.668888:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828556.668890:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828556.668894:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828556.668898:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828556.668901:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828556.668904:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828556.668908:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.668910:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.668913:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.668916:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828556.668919:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828556.668923:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828556.668926:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828556.668930:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828556.668934:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828556.668939:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.668942:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828556.668945:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828556.668955:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828556.668958:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828556.668963:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828556.668965:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.668968:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828556.668971:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828556.668976:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828556.668984:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828556.668992:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828556.669002:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327255936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828556.669015:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828556.669019:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828556.669038:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327255936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828556.669065:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828556.669068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828556.669072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828556.669074:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828556.669076:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828556.669079:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828556.669082:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2704
00000400:00000200:2.0:1787828556.669085:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828556.669088:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828556.669091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828556.669094:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669097:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.669099:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669102:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828556.669104:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828556.669106:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828556.669108:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828556.669110:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828556.669113:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828556.669115:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828556.669118:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669120:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.669122:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669123:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828556.669126:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828556.669129:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669131:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828556.669134:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828556.669140:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828556.669143:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669146:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828556.669151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828556.669154:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669157:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828556.669161:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828556.669165:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828556.669173:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828556.669176:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669182:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828556.669188:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828556.669191:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828556.669199:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828556.669208:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828556.669224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327256064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828556.669289:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828556.669293:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828556.669302:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327256064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828556.669328:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828556.669334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828556.669344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828556.669346:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828556.669350:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828556.669359:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828556.669363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2704
00000400:00000200:2.0:1787828556.669367:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828556.669370:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828556.669372:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828556.669379:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669384:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.669387:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669394:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828556.669397:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828556.669401:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828556.669404:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828556.669410:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828556.669418:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828556.669426:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828556.669438:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669443:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.669454:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669463:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828556.669468:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828556.669497:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669502:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828556.669505:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828556.669511:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828556.669517:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669519:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828556.669522:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828556.669528:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669534:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828556.669540:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828556.669549:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828556.669556:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828556.669558:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669563:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828556.669570:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828556.669575:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828556.669585:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828556.669592:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828556.669605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327256192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828556.669635:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828556.669641:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828556.669650:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327256192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828556.669696:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828556.669700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828556.669703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828556.669709:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828556.669715:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828556.669722:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828556.669732:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2704
00000400:00000200:2.0:1787828556.669745:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828556.669753:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828556.669759:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828556.669767:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669773:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787828556.669780:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828556.669782:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669789:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828556.669792:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828556.669794:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828556.669796:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828556.669798:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828556.669801:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828556.669804:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828556.669817:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669826:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828556.669834:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669840:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828556.669850:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828556.669856:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828556.669865:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828556.669869:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828556.669876:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828556.669882:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669888:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828556.669892:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828556.669898:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669910:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828556.669924:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828556.669931:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828556.669938:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828556.669941:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828556.669947:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828556.669953:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828556.669960:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828556.669972:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828556.669980:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828556.669997:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327256320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828556.670020:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828556.670029:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828556.670037:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327256320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828556.670067:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828556.670075:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2714)
00000800:00000200:0.1:1787828556.670191:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828556.670421:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828556.670757:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828557.372747:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297377453, at 4297377472
00000100:00000010:2.0:1787828557.372762:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000020:00000001:2.0:1787828557.500931:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828557.500937:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828557.500940:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828558.524958:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828558.524963:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828558.524967:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828560.572997:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828560.573003:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828560.573008:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828561.596843:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828561.596848:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828561.596853:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828561.788796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828561.788807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828561.788810:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828561.788815:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828561.788821:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828561.788832:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2709
00000400:00000200:2.0:1787828561.788842:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828561.788849:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828561.788853:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828561.788857:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.788860:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.788863:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.788866:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828561.788870:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828561.788873:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828561.788875:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828561.788879:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828561.788882:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828561.788884:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828561.788887:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.788890:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.788892:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.788894:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828561.788898:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828561.788902:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828561.788904:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828561.788907:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828561.788911:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828561.788915:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.788918:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828561.788922:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828561.788932:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828561.788937:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828561.788959:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828561.788961:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.788965:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828561.788967:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828561.788972:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828561.788981:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828561.788997:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828561.789025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327256448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828561.789040:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828561.789046:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828561.789053:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327256448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828561.789101:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828561.789104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828561.789107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828561.789108:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828561.789110:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828561.789112:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828561.789114:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2709
00000400:00000200:2.0:1787828561.789118:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828561.789120:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828561.789122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828561.789125:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789128:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.789130:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789133:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828561.789134:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828561.789135:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828561.789137:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828561.789138:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828561.789140:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828561.789142:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828561.789145:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789147:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.789165:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789169:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828561.789175:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828561.789192:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789201:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828561.789209:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828561.789233:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828561.789241:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.789248:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828561.789254:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828561.789263:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.789268:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828561.789275:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828561.789283:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828561.789292:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828561.789295:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.789301:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828561.789307:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828561.789314:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828561.789321:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828561.789330:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828561.789343:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327256576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828561.789363:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828561.789368:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828561.789378:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327256576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828561.789393:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828561.789401:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828561.789408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828561.789413:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828561.789416:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828561.789423:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828561.789430:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2709
00000400:00000200:2.0:1787828561.789460:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828561.789470:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828561.789476:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828561.789485:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789494:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.789497:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789505:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828561.789510:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828561.789512:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828561.789515:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828561.789526:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828561.789535:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828561.789541:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828561.789554:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789565:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.789574:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789580:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828561.789587:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828561.789599:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789605:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828561.789610:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828561.789618:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828561.789624:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.789630:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828561.789636:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828561.789642:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.789648:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828561.789685:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828561.789690:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828561.789694:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828561.789696:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.789698:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828561.789701:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828561.789706:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828561.789715:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000800:00000200:0.1:1787828561.789723:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828561.789725:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828561.789731:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00100000:2.0:1787828561.789740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327256704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828561.789759:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828561.789765:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828561.789793:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327256704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828561.789813:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828561.789822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828561.789850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828561.789854:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828561.789859:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828561.789868:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828561.789875:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2709
00000400:00000200:2.0:1787828561.789884:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828561.789896:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828561.789904:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828561.789914:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789920:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.789929:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789938:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828561.789946:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828561.789950:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828561.789957:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828561.789960:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828561.789964:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828561.789971:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828561.789977:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789983:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828561.789989:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828561.789992:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828561.789995:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828561.789998:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828561.790001:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828561.790006:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828561.790012:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828561.790021:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.790029:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828561.790032:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828561.790037:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.790041:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828561.790046:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828561.790053:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828561.790060:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828561.790064:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828561.790067:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828561.790072:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828561.790078:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828561.790088:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828561.790096:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828561.790108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327256832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828561.790147:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828561.790171:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828561.790185:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327256832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828561.790219:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828561.790229:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2719)
00000800:00000200:0.1:1787828561.790518:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828561.790520:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828561.790821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828561.790824:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828561.791097:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828561.791099:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1787828562.492751:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297382573, at 4297382592
00000100:00000010:2.0:1787828562.492765:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000020:00000001:2.0:1787828563.644903:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828563.644908:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828563.644913:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828564.668842:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828564.668845:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828564.668848:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828566.716824:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828566.716828:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828566.716830:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828566.909363:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828566.909370:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828566.910084:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828566.910086:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828566.910279:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828566.910281:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828566.910509:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828566.910512:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787828567.612799:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297387692, at 4297387712
00000100:00000010:2.0:1787828567.612818:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000020:00000001:2.0:1787828567.740901:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828567.740905:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828567.740909:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828569.788915:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828569.788921:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828569.788926:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828570.812999:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828570.813010:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828570.813018:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828572.028790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828572.028801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828572.028804:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828572.028808:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828572.028813:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828572.028820:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2719
00000400:00000200:2.0:1787828572.028828:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828572.028834:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828572.028836:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828572.028839:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.028841:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.028843:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.028844:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828572.028847:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828572.028849:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828572.028851:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828572.028854:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828572.028855:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828572.028857:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828572.028859:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.028861:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.028862:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.028863:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828572.028865:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828572.028868:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828572.028869:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828572.028872:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828572.028875:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828572.028876:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.028878:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828572.028880:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828572.028888:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787828572.028890:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828572.028893:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828572.028894:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.028897:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828572.028899:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828572.028902:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828572.028908:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828572.028913:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828572.028921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327257472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828572.028929:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828572.028933:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828572.028938:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327257472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828572.028962:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828572.028966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828572.028969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828572.028970:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828572.028972:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828572.028975:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828572.028979:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2719
00000400:00000200:2.0:1787828572.028983:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828572.028988:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828572.028991:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828572.028996:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.028999:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.029002:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029006:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828572.029012:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828572.029015:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828572.029018:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828572.029021:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828572.029026:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828572.029030:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828572.029038:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029041:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.029045:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029049:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828572.029055:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828572.029062:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029069:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828572.029072:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828572.029078:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828572.029083:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029086:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828572.029089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828572.029095:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029098:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828572.029105:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828572.029111:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828572.029115:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828572.029117:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029120:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828572.029121:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828572.029124:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828572.029133:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828572.029138:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828572.029147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327257600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828572.029162:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828572.029168:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828572.029178:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327257600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828572.029200:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828572.029207:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828572.029214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828572.029218:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828572.029222:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828572.029229:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828572.029235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2719
00000400:00000200:2.0:1787828572.029242:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828572.029245:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828572.029247:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828572.029249:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029253:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.029258:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029262:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828572.029265:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828572.029268:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828572.029270:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828572.029274:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828572.029277:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828572.029282:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828572.029286:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029290:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.029296:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029301:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828572.029304:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787828572.029307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787828572.029310:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.1:1787828572.029312:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787828572.029313:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828572.029319:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828572.029322:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828572.029327:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029332:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828572.029333:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828572.029336:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029340:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828572.029347:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828572.029351:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828572.029354:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828572.029355:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029356:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828572.029358:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828572.029360:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828572.029365:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828572.029385:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828572.029395:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327257728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828572.029428:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828572.029433:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828572.029443:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327257728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828572.029455:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828572.029458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828572.029465:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828572.029470:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828572.029473:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828572.029476:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828572.029480:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2719
00000400:00000200:2.0:1787828572.029485:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828572.029489:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828572.029492:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828572.029495:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029500:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.029506:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029513:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828572.029521:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828572.029526:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828572.029531:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828572.029536:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828572.029540:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828572.029547:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828572.029557:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029563:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828572.029571:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029577:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828572.029585:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828572.029595:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828572.029603:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828572.029608:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828572.029612:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828572.029617:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029622:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828572.029626:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828572.029631:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029635:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828572.029643:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828572.029648:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828572.029653:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828572.029656:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828572.029712:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828572.029743:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828572.029747:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828572.029756:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828572.029764:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828572.029772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327257856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828572.029800:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828572.029806:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828572.029816:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327257856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828572.029842:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828572.029847:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2729)
00000800:00000200:0.1:1787828572.030023:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828572.030036:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787828572.030278:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828572.030280:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787828572.030495:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828572.030496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000040:2.0:1787828572.732754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297392812, at 4297392832
00000100:00000010:2.0:1787828572.732780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000020:00000001:2.0:1787828572.860963:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828572.860968:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828572.860972:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828573.884909:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828573.884913:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828573.884916:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828575.932859:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828575.932872:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828575.932877:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828576.956914:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828576.956921:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828576.956928:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828577.148766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828577.148786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828577.148788:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828577.148792:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828577.148796:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828577.148802:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2724
00000400:00000200:2.0:1787828577.148809:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828577.148816:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828577.148819:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828577.148823:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148825:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.148827:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148829:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828577.148832:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828577.148835:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828577.148837:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828577.148840:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828577.148842:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828577.148844:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828577.148846:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148847:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.148849:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148850:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828577.148852:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828577.148855:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148856:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828577.148859:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828577.148861:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828577.148864:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.148865:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828577.148867:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828577.148876:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828577.148878:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828577.148881:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828577.148882:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.148885:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828577.148887:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828577.148890:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828577.148897:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828577.148902:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828577.148910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327257984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828577.148919:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828577.148923:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828577.148927:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327257984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828577.148957:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828577.148959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828577.148962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828577.148963:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828577.148964:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828577.148966:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828577.148969:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2724
00000400:00000200:2.0:1787828577.148972:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828577.148975:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828577.148977:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828577.148979:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148981:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.148984:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.148989:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828577.148991:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828577.148995:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828577.148997:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828577.149000:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828577.149003:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828577.149006:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828577.149013:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149017:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.149021:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149025:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828577.149053:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828577.149060:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149066:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828577.149073:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828577.149082:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828577.149084:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149086:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828577.149105:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828577.149116:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149120:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828577.149129:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828577.149136:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828577.149145:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828577.149150:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149154:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828577.149161:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828577.149174:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828577.149191:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828577.149205:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828577.149227:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327258112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828577.149256:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828577.149267:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828577.149285:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327258112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828577.149357:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828577.149380:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828577.149386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828577.149388:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828577.149389:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828577.149392:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828577.149397:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2724
00000400:00000200:2.0:1787828577.149405:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828577.149412:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828577.149416:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828577.149422:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149443:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.149453:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149463:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828577.149471:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828577.149477:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828577.149482:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828577.149488:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828577.149494:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828577.149502:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828577.149512:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149521:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.149530:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149535:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828577.149544:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828577.149554:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149564:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828577.149567:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828577.149572:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828577.149579:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149584:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828577.149589:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828577.149597:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149602:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828577.149608:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828577.149611:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828577.149620:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828577.149624:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149631:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.1:1787828577.149632:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787828577.149634:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828577.149641:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828577.149643:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
02000000:00000010:2.0:1787828577.149650:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828577.149659:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828577.149667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327258240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828577.149755:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828577.149759:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828577.149765:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327258240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828577.149793:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828577.149796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828577.149800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828577.149803:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828577.149806:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828577.149809:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828577.149814:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2724
00000400:00000200:2.0:1787828577.149822:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828577.149827:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828577.149830:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828577.149837:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149840:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.149847:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149853:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828577.149860:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828577.149864:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828577.149867:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828577.149870:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828577.149875:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828577.149879:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828577.149883:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149888:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828577.149895:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149902:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828577.149910:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828577.149918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828577.149924:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828577.149930:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828577.149935:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828577.149942:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149948:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828577.149954:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828577.149962:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149967:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828577.149976:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828577.149982:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828577.149990:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828577.149994:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828577.149998:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828577.150002:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828577.150006:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828577.150013:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828577.150017:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828577.150027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327258368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828577.150044:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828577.150049:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828577.150057:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327258368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828577.150072:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828577.150079:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2734)
00000800:00000200:0.1:1787828577.150268:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828577.150270:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787828577.150651:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828577.150654:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787828577.150959:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828577.150961:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1787828577.852803:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297397933, at 4297397952
00000100:00000010:2.0:1787828577.852817:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000020:00000001:2.0:1787828579.004946:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828579.004953:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828579.004958:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828580.028805:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828580.028810:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828580.028814:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828582.076876:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828582.076881:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828582.076885:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828582.268811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828582.268828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828582.268833:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828582.268839:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828582.268846:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828582.268858:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2729
00000400:00000200:2.0:1787828582.268872:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828582.268883:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828582.268889:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828582.268894:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.268897:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.268900:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.268903:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828582.268908:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828582.268910:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828582.268914:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828582.268918:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828582.268921:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828582.268923:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828582.268926:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.268929:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.268931:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.268932:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828582.268936:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828582.268941:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828582.268944:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828582.268947:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828582.268952:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828582.268955:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.268958:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828582.268961:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828582.268986:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828582.268991:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828582.268996:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828582.268998:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269002:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828582.269006:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828582.269012:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828582.269023:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828582.269032:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828582.269044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327258496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828582.269056:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828582.269062:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828582.269067:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327258496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828582.269095:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828582.269099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828582.269103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828582.269104:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828582.269106:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828582.269110:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828582.269112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2729
00000400:00000200:2.0:1787828582.269116:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828582.269120:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828582.269122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828582.269126:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269128:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.269130:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269132:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828582.269134:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828582.269136:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828582.269137:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828582.269139:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828582.269141:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828582.269143:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828582.269145:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269147:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.269149:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269151:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828582.269154:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828582.269156:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269160:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828582.269162:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828582.269169:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828582.269171:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269173:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828582.269179:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828582.269183:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269185:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828582.269189:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828582.269198:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828582.269207:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828582.269213:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269217:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828582.269219:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828582.269222:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828582.269228:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828582.269240:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828582.269256:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327258624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828582.269289:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828582.269296:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828582.269336:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327258624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828582.269351:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828582.269356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828582.269370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828582.269377:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828582.269381:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828582.269394:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828582.269399:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2729
00000400:00000200:2.0:1787828582.269413:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828582.269427:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828582.269431:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828582.269442:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269452:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.269460:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269468:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828582.269479:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828582.269488:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828582.269500:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828582.269510:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828582.269520:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828582.269531:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828582.269547:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269557:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.269571:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269582:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828582.269592:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828582.269608:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828582.269622:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828582.269632:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828582.269641:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828582.269653:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269687:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828582.269690:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828582.269693:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269696:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828582.269702:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828582.269711:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828582.269728:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828582.269733:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.269746:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828582.269752:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828582.269759:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828582.269768:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000010:2.0:1787828582.269772:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000800:00000200:0.1:1787828582.269773:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1787828582.269780:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828582.269796:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327258752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828582.269823:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828582.269832:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828582.269845:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327258752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828582.269930:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828582.269941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828582.269961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828582.269969:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828582.269977:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828582.269987:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828582.269994:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2729
00000400:00000200:2.0:1787828582.270005:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828582.270013:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828582.270017:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828582.270027:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.270035:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.270042:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.270061:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828582.270073:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828582.270078:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828582.270083:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828582.270089:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828582.270099:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828582.270106:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828582.270117:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.270129:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828582.270141:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828582.270147:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828582.270156:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828582.270171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828582.270177:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828582.270183:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828582.270193:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828582.270209:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.270217:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828582.270227:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828582.270238:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.270247:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828582.270260:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828582.270268:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828582.270278:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828582.270282:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828582.270292:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828582.270316:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828582.270322:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828582.270332:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828582.270343:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828582.270366:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327258880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828582.270391:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828582.270397:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828582.270413:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327258880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828582.270443:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828582.270456:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2740)
00000800:00000200:0.1:1787828582.270605:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828582.270607:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828582.270813:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828582.270815:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828582.270988:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828582.270990:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1787828582.972818:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297403053, at 4297403072
00000100:00000010:2.0:1787828582.972838:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000020:00000001:2.0:1787828583.100834:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828583.100840:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828583.100845:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828585.148914:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828585.148919:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828585.148923:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828586.172952:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828586.172957:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828586.172961:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828587.389293:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828587.389297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828587.389761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828587.389778:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828587.389983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828587.389984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828587.390163:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828587.390164:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787828588.092756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297408172, at 4297408192
00000100:00000010:2.0:1787828588.092792:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000020:00000001:2.0:1787828588.220810:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828588.220815:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828588.220819:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828589.244886:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828589.244890:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828589.244894:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828590.972764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828590.972772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828590.972774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828590.972779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828590.972783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828590.972787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828590.972789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828590.972791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828590.972794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828591.292850:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828591.292854:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828591.292857:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828591.996740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828591.996750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828591.996753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828591.996760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828591.996765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828591.996769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828591.996772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828591.996776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828591.996780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828592.316812:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828592.316816:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828592.316819:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828592.509551:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828592.509566:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828592.510078:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828592.510080:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828592.510304:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828592.510305:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828592.510492:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828592.510494:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1787828593.020735:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828593.020741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828593.020743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828593.020748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828593.020751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828593.020753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828593.020755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828593.020757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828593.020760:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828593.212739:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297413293, at 4297413312
00000100:00000010:2.0:1787828593.212753:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000001:2.0:1787828594.044748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828594.044754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828594.044756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828594.044760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828594.044762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828594.044765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828594.044767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828594.044770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828594.044772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828594.364879:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828594.364883:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828594.364887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828595.068740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828595.068747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828595.068748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828595.068752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828595.068754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828595.068757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828595.068759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828595.068761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828595.068763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828595.388823:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828595.388827:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828595.388830:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828596.092735:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828596.092741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828596.092743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828596.092747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828596.092750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828596.092753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828596.092754:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828596.092757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828596.092759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828597.116758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828597.116765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828597.116767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828597.116772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828597.116776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828597.116790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828597.116793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828597.116801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828597.116804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828597.436852:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828597.436859:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828597.436865:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828597.629326:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828597.629331:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828597.629862:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828597.629864:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828597.630085:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828597.630087:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828597.630308:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828597.630310:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787828598.140747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828598.140756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828598.140759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828598.140765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828598.140770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828598.140774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828598.140777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828598.140781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828598.140784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828598.332795:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297418412, at 4297418432
00000100:00000010:2.0:1787828598.332814:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fba0.
00000100:00000001:2.0:1787828599.164747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828599.164756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828599.164758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828599.164764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828599.164768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828599.164771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828599.164774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828599.164777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828599.164779:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828600.508887:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828600.508892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828600.508894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828602.749260:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828602.749265:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828602.749785:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828602.749787:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828602.749970:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828602.749971:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828602.750214:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828602.750216:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828603.452801:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297423532, at 4297423552
00000100:00000010:2.0:1787828603.452816:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000020:00000001:2.0:1787828603.580863:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828603.580868:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828603.580872:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828606.652844:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828606.652848:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828606.652852:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828607.868788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828607.868799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828607.868801:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828607.868806:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828607.868811:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828607.868818:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2755
00000400:00000200:2.0:1787828607.868827:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828607.868834:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828607.868837:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828607.868840:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.868843:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.868845:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.868847:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828607.868851:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828607.868852:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828607.868855:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828607.868859:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828607.868860:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828607.868862:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828607.868864:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.868865:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.868867:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.868868:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828607.868871:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828607.868874:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828607.868875:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828607.868878:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828607.868881:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828607.868884:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.868887:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828607.868889:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828607.868899:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828607.868902:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828607.868906:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828607.868907:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.868910:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828607.868913:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828607.868918:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828607.868926:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828607.868933:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828607.868942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327261056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828607.868951:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828607.868955:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828607.868960:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327261056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828607.868979:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828607.868982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828607.868985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828607.868986:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828607.868987:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828607.868989:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828607.868991:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2755
00000400:00000200:2.0:1787828607.868993:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828607.868995:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828607.868997:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828607.869000:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869002:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.869004:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869006:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828607.869010:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828607.869012:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828607.869015:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828607.869017:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828607.869020:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828607.869022:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828607.869026:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869029:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.869033:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869036:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828607.869041:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828607.869045:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869049:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828607.869052:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828607.869058:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828607.869064:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869068:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828607.869074:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828607.869078:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869081:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828607.869084:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828607.869087:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828607.869090:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828607.869093:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869094:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828607.869095:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828607.869096:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828607.869101:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828607.869105:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828607.869112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327261184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828607.869133:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828607.869137:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828607.869141:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327261184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828607.869154:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828607.869158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828607.869162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828607.869164:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828607.869167:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828607.869172:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828607.869177:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2755
00000400:00000200:2.0:1787828607.869181:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828607.869188:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828607.869191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828607.869195:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869200:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.869202:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869204:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828607.869208:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828607.869211:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828607.869213:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828607.869215:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828607.869216:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828607.869218:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828607.869225:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869228:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.869229:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869233:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828607.869239:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828607.869243:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869247:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828607.869249:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828607.869253:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828607.869257:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869261:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828607.869264:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828607.869271:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869276:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828607.869283:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828607.869289:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828607.869297:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828607.869301:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869307:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828607.869311:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828607.869315:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828607.869322:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828607.869328:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828607.869333:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828607.869338:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00100000:2.0:1787828607.869340:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327261312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828607.869353:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828607.869356:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828607.869360:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327261312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828607.869373:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828607.869376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828607.869380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828607.869383:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828607.869394:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828607.869399:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828607.869403:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2755
00000400:00000200:2.0:1787828607.869410:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828607.869417:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828607.869421:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828607.869427:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869430:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.869435:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869437:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828607.869440:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828607.869442:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828607.869444:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828607.869447:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828607.869451:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828607.869456:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828607.869459:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869464:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828607.869465:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869466:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828607.869468:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828607.869470:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828607.869474:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828607.869477:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828607.869481:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828607.869485:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869488:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828607.869492:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828607.869494:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869498:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828607.869505:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828607.869510:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828607.869515:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828607.869516:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828607.869520:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828607.869523:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828607.869525:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828607.869531:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828607.869536:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828607.869543:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327261440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828607.869555:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828607.869559:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828607.869564:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327261440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828607.869584:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828607.869587:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2765)
00000800:00000200:0.1:1787828607.869817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828607.869835:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828607.870031:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828607.870032:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828607.870239:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828607.870241:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828608.572742:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297428652, at 4297428672
00000100:00000010:2.0:1787828608.572758:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000020:00000001:2.0:1787828609.724821:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828609.724825:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828609.724829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828612.476767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828612.476776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828612.476779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828612.476785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828612.476790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828612.476794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828612.476798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828612.476801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828612.476805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828612.796843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828612.796847:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828612.796850:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828612.989458:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828612.989466:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828612.990053:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828612.990055:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828612.990405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828612.990408:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828612.990754:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828612.990757:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787828613.500795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828613.500804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828613.500806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828613.500811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828613.500814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828613.500818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828613.500821:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828613.500825:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828613.500828:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828613.692764:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297433772, at 4297433792
00000100:00000010:2.0:1787828613.692792:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000001:2.0:1787828614.524758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828614.524767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828614.524771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828614.524778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828614.524782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828614.524786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828614.524790:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828614.524793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828614.524799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828615.548795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828615.548805:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828615.548807:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828615.548814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828615.548819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828615.548824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828615.548827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828615.548831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828615.548834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828615.868860:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828615.868864:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828615.868868:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828616.572756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828616.572763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828616.572765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828616.572771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828616.572774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828616.572779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828616.572782:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828616.572787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828616.572791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828617.596754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828617.596765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828617.596767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828617.596774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828617.596779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828617.596783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828617.596786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828617.596790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828617.596794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828618.109405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828618.109413:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787828618.110046:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828618.110047:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787828618.110318:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828618.110320:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787828618.110538:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828618.110540:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1787828618.620788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828618.620797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828618.620799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828618.620804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828618.620808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828618.620811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828618.620813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828618.620816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828618.620819:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828618.812750:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297438892, at 4297438912
00000100:00000010:2.0:1787828618.812770:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000020:00000001:2.0:1787828618.940848:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828618.940853:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828618.940859:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828619.644745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828619.644754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828619.644756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828619.644761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828619.644765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828619.644769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828619.644771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828619.644773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828619.644776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828620.668743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828620.668754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828620.668756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828620.668764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828620.668769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828620.668774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828620.668779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828620.668783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828620.668787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828621.692755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828621.692765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828621.692768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828621.692776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828621.692781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828621.692785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828621.692789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828621.692792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828621.692797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828622.012847:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828622.012853:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828622.012858:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828622.716724:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828622.716731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828622.716733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828622.716738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828622.716742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828622.716745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828622.716748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828622.716750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828622.716752:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828623.229451:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828623.229733:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828623.229901:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828623.230078:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828623.740726:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828623.740736:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828623.740740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828623.740746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828623.740757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828623.740761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828623.740763:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828623.740766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828623.740769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828623.932729:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297444012, at 4297444032
00000100:00000010:2.0:1787828623.932739:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f660.
00000100:00000001:2.0:1787828624.764738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828624.764746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828624.764748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828624.764753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828624.764757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828624.764761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828624.764764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828624.764767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828624.764769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828625.788745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828625.788751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828625.788753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828625.788757:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828625.788760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828625.788763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828625.788764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828625.788766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828625.788768:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828626.812795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828626.812801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828626.812802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828626.812806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828626.812809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828626.812813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828626.812815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828626.812818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828626.812822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828627.836778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828627.836787:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828627.836791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828627.836797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828627.836800:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828627.836804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828627.836806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828627.836809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828627.836812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828628.349424:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828628.349451:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828628.350438:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828628.350440:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828628.350645:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828628.350647:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828628.350874:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828628.350876:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787828628.860741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828628.860748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828628.860750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828628.860756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828628.860761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828628.860764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828628.860767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828628.860770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828628.860772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828629.052731:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297449133, at 4297449152
00000100:00000010:2.0:1787828629.052743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3ff00.
00000100:00000001:2.0:1787828629.884804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828629.884814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828629.884818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828629.884825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828629.884830:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828629.884835:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828629.884839:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828629.884842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828629.884846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828630.908746:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828630.908754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828630.908756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828630.908761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828630.908765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828630.908769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828630.908771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828630.908773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828630.908776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828631.932806:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828631.932815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828631.932817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828631.932822:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828631.932826:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828631.932829:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828631.932831:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828631.932833:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828631.932836:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828632.956796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828632.956804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828632.956806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828632.956811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828632.956832:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828632.956837:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828632.956840:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828632.956844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828632.956847:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828633.469288:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828633.469294:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828633.469844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828633.469846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828633.470064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828633.470066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828633.470252:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828633.470254:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787828633.980818:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828633.980829:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828633.980831:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828633.980838:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828633.980841:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828633.980846:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828633.980848:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828633.980851:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828633.980853:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828634.172737:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297454252, at 4297454272
00000100:00000010:2.0:1787828634.172756:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f900.
00000100:00000001:2.0:1787828635.004739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828635.004747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828635.004749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828635.004755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828635.004759:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828635.004763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828635.004765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828635.004768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828635.004770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828636.028748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828636.028756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828636.028758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828636.028764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828636.028769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828636.028774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828636.028778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828636.028782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828636.028788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828637.052751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828637.052760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828637.052763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828637.052768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828637.052772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828637.052775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828637.052778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828637.052780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828637.052783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828638.076736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828638.076744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828638.076746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828638.076751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828638.076755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828638.076758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828638.076761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828638.076763:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828638.076765:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828638.589341:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828638.589345:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828638.589935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828638.589936:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828638.590155:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828638.590158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828638.590350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828638.590352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787828639.100717:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828639.100722:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828639.100724:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828639.100728:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828639.100731:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828639.100734:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828639.100737:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828639.100738:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828639.100741:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828639.292721:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297459372, at 4297459392
00000100:00000010:2.0:1787828639.292731:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f4e0.
00000100:00000001:2.0:1787828640.124749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828640.124757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828640.124759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828640.124765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828640.124768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828640.124772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828640.124774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828640.124776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828640.124778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828641.148748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828641.148757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828641.148759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828641.148764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828641.148768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828641.148771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828641.148773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828641.148787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828641.148789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828641.468846:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828641.468851:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828641.468854:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828642.172740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828642.172746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828642.172748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828642.172752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828642.172755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828642.172758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828642.172760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828642.172762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828642.172765:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828643.196727:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828643.196733:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828643.196735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828643.196739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828643.196742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828643.196745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828643.196748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828643.196750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828643.196753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828643.709541:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828643.709565:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828643.710230:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828643.710232:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828643.710470:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828643.710472:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828643.710786:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828643.710789:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787828644.220764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828644.220769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828644.220770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828644.220774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828644.220777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828644.220780:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828644.220782:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828644.220784:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828644.220786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828644.412745:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297464493, at 4297464512
00000100:00000010:2.0:1787828644.412763:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f720.
00000020:00000001:2.0:1787828644.540878:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828644.540883:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828644.540887:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828645.244749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828645.244757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828645.244760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828645.244765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828645.244768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828645.244773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828645.244775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828645.244778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828645.244780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828646.268873:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828646.268883:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828646.268885:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828646.268889:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828646.268893:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828646.268897:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828646.268905:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828646.268908:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828646.268913:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828647.292787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828647.292793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828647.292794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828647.292798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828647.292801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828647.292804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828647.292806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828647.292808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828647.292810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828647.612836:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828647.612839:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828647.612842:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828648.316727:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828648.316733:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828648.316735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828648.316738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828648.316740:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828648.316743:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828648.316745:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828648.316747:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828648.316749:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828648.829644:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828648.829655:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828648.830246:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828648.830248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828648.830551:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828648.830553:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828648.830790:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828648.830792:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1787828649.340827:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828649.340856:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828649.340860:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828649.340868:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828649.340874:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828649.340895:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828649.340899:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828649.340903:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828649.340907:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828649.532758:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297469613, at 4297469632
00000100:00000010:2.0:1787828649.532774:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd80.
00000100:00000001:2.0:1787828650.364767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828650.364776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828650.364779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828650.364786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828650.364791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828650.364796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828650.364800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828650.364803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828650.364808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828650.684927:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828650.684935:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828650.684940:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828651.388792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828651.388800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828651.388803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828651.388808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828651.388812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828651.388817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828651.388833:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828651.388836:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828651.388839:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828652.412777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828652.412783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828652.412784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828652.412788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828652.412791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828652.412793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828652.412795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828652.412797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828652.412799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828653.436768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828653.436776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828653.436778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828653.436784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828653.436787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828653.436791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828653.436794:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828653.436796:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828653.436799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828653.756891:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828653.756896:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828653.756900:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787828653.949522:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828653.949531:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828653.949964:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828653.949966:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828653.950199:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828653.950201:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828653.950396:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828653.950398:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000001:2.0:1787828654.460741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828654.460749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828654.460751:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828654.460756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828654.460760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828654.460763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828654.460765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828654.460768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828654.460770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828654.652736:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297474732, at 4297474752
00000100:00000010:2.0:1787828654.652763:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fb40.
00000100:00000001:2.0:1787828655.484847:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828655.484855:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828655.484857:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828655.484863:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828655.484866:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828655.484870:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828655.484873:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828655.484876:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828655.484879:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828656.508729:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828656.508734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828656.508736:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828656.508740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828656.508742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828656.508745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828656.508747:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828656.508749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828656.508751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828656.828851:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828656.828857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828656.828861:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828657.532752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828657.532759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828657.532788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828657.532793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828657.532798:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828657.532801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828657.532803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828657.532805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828657.532808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828658.556732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828658.556739:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828658.556741:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828658.556746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828658.556749:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828658.556753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828658.556755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828658.556758:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828658.556760:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069223:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828659.069226:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828659.069230:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828659.069232:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2806
00000400:00000200:3.0:1787828659.069238:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828659.069242:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828659.069245:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828659.069250:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069254:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828659.069257:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069259:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828659.069262:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828659.069264:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828659.069267:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828659.069271:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828659.069276:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828659.069280:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828659.069288:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069292:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828659.069297:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069300:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828659.069305:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828659.069320:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069330:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828659.069336:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828659.069357:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828659.069368:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069372:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828659.069382:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828659.069392:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069396:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828659.069408:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828659.069413:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828659.069422:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828659.069423:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069426:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828659.069431:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828659.069434:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828659.069443:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828659.069452:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828659.069461:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327266304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828659.069471:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828659.069475:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828659.069488:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327266304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828659.069502:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828659.069518:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828659.069526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828659.069531:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828659.069535:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828659.069539:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828659.069546:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2806
00000400:00000200:3.0:1787828659.069552:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828659.069557:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828659.069566:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828659.069570:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069574:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828659.069580:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069586:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828659.069592:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828659.069596:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828659.069601:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828659.069604:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828659.069609:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828659.069613:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828659.069625:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069632:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828659.069638:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069645:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828659.069650:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828659.069707:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069710:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828659.069712:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828659.069716:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828659.069719:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069725:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828659.069729:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828659.069735:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069738:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828659.069749:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828659.069753:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828659.069760:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828659.069761:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.069785:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828659.069787:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828659.069789:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828659.069798:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828659.069805:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828659.069833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327266432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828659.069846:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828659.069853:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828659.069864:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327266432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828659.069880:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828659.069892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828659.069898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828659.069902:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828659.069903:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828659.069909:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828659.069915:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2806
00000400:00000200:3.0:1787828659.069926:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828659.069938:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828659.069945:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828659.069964:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069968:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828659.069970:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.069974:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828659.069977:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828659.069981:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828659.069986:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828659.069991:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828659.069995:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828659.070002:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828659.070010:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.070015:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828659.070021:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828659.070026:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828659.070029:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828659.070033:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828659.070040:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828659.070043:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828659.070047:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828659.070054:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.070061:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828659.070065:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828659.070068:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828659.070073:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828659.070081:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000800:00000200:0.1:1787828659.070087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787828659.070088:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828659.070095:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828659.070098:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828659.070099:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000100:00000001:3.0:1787828659.070100:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828659.070104:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828659.070108:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828659.070114:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828659.070120:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828659.070130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327266560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828659.070140:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828659.070145:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828659.070153:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327266560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828659.070169:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828659.070180:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2816)
00000800:00000200:0.1:1787828659.070932:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828659.070956:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787828659.071482:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828659.071487:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787828659.072061:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828659.072065:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000100:00000001:2.0:1787828659.580737:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828659.580742:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828659.580751:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828659.580755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828659.580758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828659.580774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828659.580776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828659.580778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828659.580781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828659.772828:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297479853, at 4297479872
00000100:00000040:2.0:1787828659.772831:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297479853, at 4297479872
00000100:00000010:3.0:1787828659.772847:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00000100:00000010:2.0:1787828659.772853:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000020:00000001:2.0:1787828659.900896:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828659.900908:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828659.900912:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828660.604741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828660.604752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828660.604766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828660.604773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828660.604778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828660.604783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828660.604786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828660.604789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828660.604793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828660.924918:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828660.924923:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828660.924927:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828661.628739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828661.628761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828661.628763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828661.628769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828661.628772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828661.628776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828661.628778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828661.628781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828661.628784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828661.948911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828661.948917:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828661.948921:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828662.652752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828662.652762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828662.652764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828662.652769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828662.652789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828662.652793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828662.652796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828662.652799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828662.652802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828662.972869:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828662.972873:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828662.972877:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828663.676719:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828663.676727:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828663.676729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828663.676746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828663.676749:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828663.676754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828663.676768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828663.676770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828663.676773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828663.996906:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828663.996910:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828663.996920:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828664.188802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828664.188813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828664.188816:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828664.188820:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828664.188825:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828664.188833:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2811
00000400:00000200:3.0:1787828664.188844:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828664.188852:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828664.188856:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828664.188860:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.188864:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.188867:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.188870:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828664.188875:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828664.188877:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828664.188880:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828664.188885:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828664.188888:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828664.188891:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828664.188895:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.188897:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.188899:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.188902:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828664.188906:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828664.188926:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828664.188928:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828664.188931:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828664.188936:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828664.188939:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.188942:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828664.188945:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828664.188954:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787828664.188958:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828664.188962:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828664.188964:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.188967:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828664.188970:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828664.188974:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828664.188981:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787828664.188988:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828664.188997:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327266688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828664.189008:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828664.189013:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828664.189019:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327266688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828664.189047:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828664.189050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828664.189054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828664.189056:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828664.189057:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828664.189060:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828664.189063:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2811
00000400:00000200:3.0:1787828664.189068:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828664.189072:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828664.189075:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828664.189079:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189081:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.189084:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189087:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828664.189091:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828664.189094:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828664.189099:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828664.189103:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828664.189107:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828664.189114:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828664.189123:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189128:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.189134:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189141:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828664.189148:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828664.189157:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189163:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828664.189169:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828664.189178:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828664.189184:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189188:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828664.189194:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828664.189199:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189202:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828664.189209:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787828664.189216:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828664.189222:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828664.189225:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189230:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828664.189235:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828664.189239:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828664.189248:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828664.189261:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828664.189273:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327266816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828664.189291:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828664.189298:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828664.189309:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327266816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828664.189325:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828664.189331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828664.189340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828664.189342:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828664.189344:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828664.189350:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828664.189361:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2811
00000400:00000200:3.0:1787828664.189370:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.1:1787828664.189376:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:3.0:1787828664.189378:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:0.1:1787828664.189380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000100:00080000:3.0:1787828664.189384:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828664.189394:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189398:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.189403:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189409:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828664.189414:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828664.189418:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828664.189423:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828664.189427:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828664.189432:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828664.189438:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828664.189445:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189450:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.189456:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189462:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828664.189467:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828664.189473:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189480:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828664.189483:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828664.189487:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828664.189492:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189497:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828664.189502:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828664.189507:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189512:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828664.189517:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828664.189522:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828664.189527:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828664.189531:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189537:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828664.189542:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828664.189547:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828664.189554:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828664.189561:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828664.189572:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327266944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828664.189592:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828664.189597:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828664.189607:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327266944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828664.189625:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828664.189630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828664.189638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828664.189643:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828664.189647:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828664.189685:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828664.189691:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2811
00000400:00000200:3.0:1787828664.189695:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828664.189699:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828664.189703:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828664.189711:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189717:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.189723:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189744:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828664.189750:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828664.189754:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828664.189758:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828664.189763:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828664.189768:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828664.189774:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828664.189782:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189787:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828664.189792:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189797:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828664.189803:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828664.189811:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828664.189816:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828664.189820:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828664.189825:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828664.189830:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189835:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828664.189840:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828664.189846:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189851:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828664.189859:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828664.189865:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828664.189870:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828664.189874:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828664.189881:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828664.189886:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828664.189890:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828664.189898:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828664.189907:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828664.189916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327267072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828664.189933:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828664.189937:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828664.189946:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327267072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828664.189969:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828664.189977:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2821)
00000800:00000200:0.1:1787828664.190064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828664.190066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787828664.190279:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828664.190280:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787828664.190537:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828664.190539:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000100:00000040:2.0:1787828664.892837:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297484972, at 4297484992
00000100:00000040:3.0:1787828664.892837:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297484972, at 4297484992
00000100:00000010:3.0:1787828664.892855:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000010:2.0:1787828664.892859:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000020:00000001:3.0:1787828665.020871:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828665.020875:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828665.020879:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828666.044846:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828666.044849:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828666.044852:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828667.068898:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828667.068904:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828667.068909:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828668.092872:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828668.092878:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828668.092882:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828669.116995:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828669.117004:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828669.117013:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828669.308798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828669.308809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828669.308812:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828669.308817:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828669.308822:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828669.308831:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2816
00000400:00000200:2.0:1787828669.308840:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828669.308847:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828669.308850:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828669.308853:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.308855:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.308857:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.308860:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828669.308864:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828669.308866:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828669.308869:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828669.308873:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828669.308874:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828669.308876:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828669.308878:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.308880:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.308882:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.308884:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828669.308887:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828669.308891:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828669.308894:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828669.308897:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828669.308902:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828669.308906:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.308909:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828669.308912:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828669.308924:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787828669.308929:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828669.308935:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828669.308937:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.308941:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828669.308944:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828669.308950:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828669.308958:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828669.308965:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828669.308975:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327267200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828669.308984:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828669.308988:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828669.308993:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327267200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828669.309014:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828669.309016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828669.309019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828669.309020:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828669.309022:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828669.309024:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828669.309026:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2816
00000400:00000200:2.0:1787828669.309028:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828669.309030:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828669.309033:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828669.309035:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309037:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.309038:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309040:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828669.309042:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828669.309042:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828669.309044:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828669.309046:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828669.309047:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828669.309048:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828669.309050:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309051:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.309052:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309053:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828669.309055:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828669.309065:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309067:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828669.309069:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828669.309074:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828669.309076:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309077:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828669.309082:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828669.309085:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309087:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828669.309092:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828669.309095:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828669.309097:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828669.309098:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309101:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828669.309102:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828669.309103:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828669.309107:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828669.309111:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828669.309119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327267328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828669.309135:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828669.309137:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828669.309145:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327267328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828669.309173:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828669.309179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828669.309191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828669.309195:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828669.309202:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828669.309207:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828669.309212:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2816
00000400:00000200:2.0:1787828669.309228:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828669.309235:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828669.309237:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828669.309247:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309255:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.309259:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309266:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828669.309273:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828669.309280:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828669.309284:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828669.309288:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828669.309301:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828669.309314:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828669.309325:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309336:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.309351:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309363:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828669.309375:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828669.309381:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309393:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828669.309396:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828669.309400:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828669.309406:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309409:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828669.309412:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828669.309424:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309426:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828669.309433:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828669.309438:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828669.309443:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828669.309450:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309461:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828669.309467:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828669.309472:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828669.309491:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828669.309502:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828669.309521:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327267456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828669.309531:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828669.309536:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828669.309559:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327267456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828669.309582:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828669.309595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828669.309607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828669.309611:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828669.309621:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828669.309628:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828669.309633:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2816
00000800:00000200:0.1:1787828669.309658:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828669.309664:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000400:00000200:2.0:1787828669.309681:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828669.309690:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828669.309707:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828669.309711:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309713:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.309715:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309716:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828669.309729:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828669.309730:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828669.309731:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828669.309732:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828669.309734:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828669.309736:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828669.309739:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309740:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828669.309743:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309744:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828669.309746:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828669.309748:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828669.309750:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828669.309751:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828669.309754:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828669.309756:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309758:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828669.309760:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828669.309761:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309763:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828669.309772:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828669.309778:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828669.309781:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828669.309783:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828669.309788:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828669.309794:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828669.309796:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828669.309806:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828669.309813:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828669.309825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327267584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828669.309864:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828669.309869:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828669.309879:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327267584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828669.309903:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828669.309913:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2827)
00000800:00000200:0.1:1787828669.310146:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828669.310147:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828669.310332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828669.310333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828669.310548:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828669.310550:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828670.012762:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297490093, at 4297490112
00000100:00000040:3.0:1787828670.012781:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297490093, at 4297490112
00000100:00000010:2.0:1787828670.012783:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:3.0:1787828670.012797:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000020:00000001:3.0:1787828670.140891:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828670.140895:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828670.140898:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828671.164911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828671.164917:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828671.164921:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828672.188864:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828672.188870:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828672.188874:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828673.212794:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828673.212798:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828673.212801:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828674.236898:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828674.236903:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828674.236907:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828674.429744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828674.429750:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828674.430432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828674.430452:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828674.430752:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828674.430755:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828674.431034:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828674.431037:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828675.132771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297495213, at 4297495232
00000100:00000040:3.0:1787828675.132771:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297495213, at 4297495232
00000100:00000010:2.0:1787828675.132790:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000100:00000010:3.0:1787828675.132796:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000020:00000001:2.0:1787828675.260866:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828675.260870:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828675.260873:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828676.284891:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828676.284897:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828676.284901:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828677.308881:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828677.308885:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828677.308899:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828678.332918:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828678.332925:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828678.332931:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828679.356885:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828679.356889:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828679.356893:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828679.548822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828679.548832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828679.548835:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828679.548839:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828679.548844:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828679.548852:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2827
00000400:00000200:2.0:1787828679.548862:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828679.548869:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828679.548874:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828679.548878:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.548881:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.548884:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.548887:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828679.548892:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828679.548895:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828679.548898:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828679.548902:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828679.548905:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828679.548909:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828679.548912:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.548924:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.548927:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.548929:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828679.548933:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828679.548937:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828679.548940:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828679.548943:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828679.548947:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828679.548949:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.548951:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828679.548954:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828679.548965:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828679.548969:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828679.548972:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828679.548974:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.548977:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828679.548979:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828679.548984:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828679.548994:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828679.549001:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828679.549010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327268224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828679.549020:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828679.549032:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828679.549038:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327268224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828679.549069:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828679.549072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828679.549075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828679.549076:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828679.549078:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828679.549080:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828679.549082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2827
00000400:00000200:2.0:1787828679.549085:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828679.549088:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828679.549090:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828679.549094:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549096:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.549098:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549099:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828679.549101:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828679.549102:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828679.549103:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828679.549106:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828679.549110:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828679.549112:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828679.549116:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549120:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.549125:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549128:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828679.549133:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828679.549139:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549145:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828679.549149:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828679.549156:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828679.549160:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549163:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828679.549171:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828679.549177:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549182:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828679.549190:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828679.549195:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828679.549202:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828679.549207:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549212:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828679.549216:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828679.549219:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828679.549226:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828679.549233:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828679.549249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327268352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828679.549270:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828679.549274:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828679.549282:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327268352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828679.549295:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828679.549299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828679.549306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828679.549309:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828679.549311:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828679.549315:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828679.549321:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2827
00000400:00000200:2.0:1787828679.549327:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828679.549334:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828679.549338:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828679.549361:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549364:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.549366:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549367:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828679.549369:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828679.549371:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828679.549376:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828679.549380:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828679.549384:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828679.549388:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828679.549393:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549396:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.549401:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549423:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828679.549429:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828679.549436:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549441:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828679.549445:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828679.549449:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828679.549454:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549457:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828679.549460:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828679.549466:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549471:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828679.549479:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000800:00000200:0.1:1787828679.549480:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787828679.549485:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.1:1787828679.549485:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
02000000:00000001:2.0:1787828679.549492:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828679.549494:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549497:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828679.549501:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828679.549503:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828679.549507:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828679.549512:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828679.549522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327268480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828679.549550:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828679.549554:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828679.549561:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327268480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828679.549585:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828679.549589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828679.549595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828679.549598:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828679.549602:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828679.549611:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828679.549619:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2827
00000400:00000200:2.0:1787828679.549649:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828679.549695:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828679.549700:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828679.549704:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549707:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.549710:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549712:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828679.549716:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828679.549719:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828679.549721:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828679.549724:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828679.549729:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828679.549734:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828679.549741:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549745:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828679.549751:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549755:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828679.549760:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828679.549767:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828679.549773:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828679.549778:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828679.549782:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828679.549787:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549791:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828679.549794:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828679.549798:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549801:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828679.549806:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828679.549810:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828679.549816:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828679.549819:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828679.549823:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828679.549825:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828679.549828:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828679.549835:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828679.549840:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828679.549850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327268608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828679.549868:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828679.549872:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828679.549880:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327268608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828679.549913:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828679.549920:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2837)
00000800:00000200:0.1:1787828679.550110:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828679.550112:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828679.550307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828679.550308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787828679.550597:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828679.550599:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:3.0:1787828680.252778:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297500333, at 4297500352
00000100:00000010:3.0:1787828680.252792:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000100:00000040:2.0:1787828680.252803:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297500333, at 4297500352
00000100:00000010:2.0:1787828680.252818:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000020:00000001:3.0:1787828680.380902:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828680.380908:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828680.380912:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828681.404909:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828681.404916:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828681.404940:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828682.428885:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828682.428890:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828682.428894:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828683.453068:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828683.453076:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828683.453083:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828684.476921:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828684.476928:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828684.476934:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828684.668824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828684.668835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828684.668838:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828684.668842:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828684.668847:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828684.668854:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2832
00000400:00000200:3.0:1787828684.668864:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828684.668870:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828684.668874:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828684.668878:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.668881:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.668884:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.668887:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828684.668892:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828684.668894:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828684.668897:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828684.668901:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828684.668904:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828684.668908:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828684.668911:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.668913:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.668916:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.668918:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828684.668922:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828684.668925:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828684.668928:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828684.668932:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828684.668936:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828684.668940:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.668942:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828684.668945:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828684.668956:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828684.668959:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828684.668964:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828684.668966:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.668969:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828684.668972:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828684.668976:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828684.668985:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787828684.668991:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828684.669001:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327268736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828684.669028:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828684.669033:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828684.669038:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327268736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828684.669064:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828684.669067:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828684.669071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828684.669073:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828684.669075:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828684.669095:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828684.669098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2832
00000400:00000200:3.0:1787828684.669104:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828684.669109:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828684.669112:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828684.669117:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669124:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.669131:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669137:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828684.669143:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828684.669147:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828684.669151:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828684.669156:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828684.669163:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828684.669169:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828684.669179:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669184:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.669191:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669197:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828684.669200:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828684.669209:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669215:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828684.669219:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828684.669238:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828684.669243:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.669247:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828684.669252:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828684.669258:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.669263:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828684.669271:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828684.669276:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828684.669282:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828684.669286:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.669292:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828684.669295:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828684.669301:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828684.669310:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828684.669317:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828684.669331:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327268864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828684.669347:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828684.669351:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828684.669361:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327268864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828684.669381:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828684.669388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828684.669398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828684.669402:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828684.669406:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828684.669412:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828684.669419:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2832
00000400:00000200:3.0:1787828684.669439:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828684.669450:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828684.669472:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828684.669478:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669484:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.669492:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787828684.669493:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828684.669499:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787828684.669499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000020:00000001:3.0:1787828684.669506:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828684.669511:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828684.669517:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828684.669537:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828684.669544:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828684.669551:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828684.669561:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669566:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.669574:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669581:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828684.669604:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828684.669614:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669623:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828684.669631:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828684.669637:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828684.669643:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.669650:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828684.669735:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828684.669741:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.669745:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828684.669751:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828684.669756:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828684.669761:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828684.669764:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.669767:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828684.669770:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828684.669773:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828684.669781:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828684.669803:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828684.669811:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327268992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828684.669838:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828684.669844:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828684.669870:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327268992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828684.669893:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828684.669899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828684.669909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828684.669912:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828684.669917:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828684.669923:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828684.669929:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2832
00000400:00000200:3.0:1787828684.669940:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828684.669949:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828684.669956:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828684.669963:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669970:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.669976:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.669983:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828684.669989:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828684.669993:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828684.669999:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828684.670004:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828684.670010:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828684.670015:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828684.670049:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.670055:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828684.670062:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828684.670068:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828684.670074:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828684.670080:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828684.670088:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828684.670094:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828684.670113:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828684.670119:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.670127:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828684.670131:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828684.670137:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.670144:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828684.670152:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787828684.670159:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828684.670166:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828684.670172:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828684.670178:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828684.670181:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828684.670186:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828684.670194:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828684.670202:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828684.670216:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327269120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828684.670233:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828684.670239:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828684.670248:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327269120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828684.670283:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828684.670292:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2842)
00000800:00000200:0.1:1787828684.670362:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828684.670365:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787828684.670805:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828684.670829:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787828684.671180:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828684.671182:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000040:2.0:1787828685.372742:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297505453, at 4297505472
00000100:00000010:2.0:1787828685.372759:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000040:3.0:1787828685.372773:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297505453, at 4297505472
00000100:00000010:3.0:1787828685.372789:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000020:00000001:2.0:1787828685.500841:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828685.500847:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828685.500851:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828686.525004:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828686.525010:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828686.525015:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828687.548912:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828687.548915:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828687.548918:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828688.572823:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828688.572827:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828688.572830:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828689.596843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828689.596847:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828689.596850:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828689.788758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828689.788774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828689.788777:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828689.788783:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828689.788789:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828689.788799:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2837
00000400:00000200:3.0:1787828689.788811:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828689.788817:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828689.788821:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828689.788824:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788827:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.788829:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788831:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828689.788834:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828689.788836:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828689.788839:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828689.788843:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828689.788845:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828689.788848:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828689.788851:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788853:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.788854:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788856:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828689.788858:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828689.788861:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788863:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828689.788865:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828689.788868:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828689.788871:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.788873:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828689.788875:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828689.788886:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828689.788890:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828689.788894:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828689.788896:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.788899:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828689.788902:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828689.788906:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828689.788915:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787828689.788922:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828689.788932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327269248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828689.788941:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828689.788946:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828689.788950:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327269248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828689.788971:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828689.788974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828689.788977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828689.788978:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828689.788979:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828689.788982:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828689.788983:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2837
00000400:00000200:3.0:1787828689.788985:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828689.788987:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828689.788989:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828689.788991:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788993:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.788995:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.788997:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828689.788999:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828689.789000:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828689.789001:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828689.789004:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828689.789006:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828689.789008:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828689.789011:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789015:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.789016:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789018:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828689.789021:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828689.789026:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789028:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828689.789030:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828689.789036:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828689.789040:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789044:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828689.789050:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828689.789054:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789057:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828689.789061:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787828689.789065:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828689.789068:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828689.789072:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789074:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828689.789077:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828689.789080:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828689.789085:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828689.789091:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828689.789098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327269376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828689.789105:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828689.789108:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828689.789115:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327269376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828689.789132:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828689.789136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828689.789139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828689.789140:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828689.789142:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828689.789143:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828689.789146:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2837
00000400:00000200:3.0:1787828689.789152:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828689.789154:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828689.789155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828689.789157:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789165:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.789169:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789173:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828689.789175:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828689.789177:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828689.789180:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828689.789184:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828689.789189:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828689.789197:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828689.789206:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789212:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.789218:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789222:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828689.789228:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828689.789234:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789237:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828689.789242:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828689.789245:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828689.789249:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789253:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828689.789256:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828689.789260:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789264:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828689.789269:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787828689.789274:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828689.789278:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828689.789281:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789284:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828689.789286:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828689.789289:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828689.789298:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828689.789305:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828689.789315:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327269504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828689.789329:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828689.789334:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828689.789344:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327269504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787828689.789344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828689.789357:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828689.789360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828689.789364:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828689.789366:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828689.789368:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828689.789375:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828689.789387:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2837
00000400:00000200:3.0:1787828689.789393:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828689.789397:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828689.789403:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828689.789409:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789414:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.789418:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789421:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828689.789427:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828689.789431:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828689.789435:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828689.789442:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828689.789446:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828689.789452:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828689.789458:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789460:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828689.789462:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789464:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828689.789467:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828689.789473:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828689.789476:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828689.789479:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828689.789483:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828689.789486:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789491:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828689.789495:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828689.789499:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789502:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828689.789507:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828689.789513:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828689.789519:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828689.789523:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828689.789526:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828689.789529:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828689.789531:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828689.789537:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828689.789543:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828689.789565:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327269632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828689.789577:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828689.789581:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828689.789587:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327269632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828689.789598:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828689.789606:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2847)
00000800:00000200:0.1:1787828689.789682:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828689.789849:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828689.790027:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787828690.492708:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297510572, at 4297510592
00000100:00000040:2.0:1787828690.492721:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297510572, at 4297510592
00000100:00000010:3.0:1787828690.492727:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000100:00000010:2.0:1787828690.492729:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000020:00000001:2.0:1787828690.620857:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828690.620861:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828690.620864:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828691.644853:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828691.644858:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828691.644862:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828692.668827:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828692.668831:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828692.668834:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828693.692895:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828693.692901:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828693.692905:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828694.716975:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828694.716982:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828694.716988:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787828694.910064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828694.910078:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828694.911275:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828694.911279:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828694.911626:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828694.911628:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828694.911982:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828694.911985:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1787828695.612731:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297515693, at 4297515712
00000100:00000010:3.0:1787828695.612748:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000100:00000040:2.0:1787828695.612767:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297515693, at 4297515712
00000100:00000010:2.0:1787828695.612780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000020:00000001:3.0:1787828695.740889:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828695.740894:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828695.740898:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828697.788877:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828697.788880:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828697.788883:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828698.813018:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828698.813028:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828698.813035:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828699.837043:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828699.837050:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828699.837073:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828700.028777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828700.028786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828700.028788:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828700.028791:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828700.028797:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828700.028803:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2847
00000400:00000200:2.0:1787828700.028811:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828700.028817:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828700.028820:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828700.028823:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028835:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.028837:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028839:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828700.028842:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828700.028844:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828700.028846:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828700.028849:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828700.028851:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828700.028853:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828700.028855:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028857:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.028858:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028860:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828700.028862:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828700.028864:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028866:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828700.028868:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828700.028872:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828700.028874:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.028876:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828700.028878:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828700.028885:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828700.028888:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828700.028891:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828700.028892:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.028895:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828700.028897:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828700.028900:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828700.028907:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828700.028913:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828700.028921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327270272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828700.028929:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828700.028933:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828700.028937:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327270272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828700.028956:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828700.028959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828700.028961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828700.028962:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828700.028963:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828700.028966:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828700.028967:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2847
00000400:00000200:2.0:1787828700.028970:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828700.028972:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828700.028973:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828700.028976:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028977:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.028980:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028982:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828700.028984:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828700.028985:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828700.028987:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828700.028989:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828700.028990:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828700.028992:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828700.028994:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.028996:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.028998:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029002:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828700.029004:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828700.029009:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029011:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828700.029014:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828700.029019:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828700.029021:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029023:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828700.029028:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828700.029034:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029038:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828700.029043:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828700.029047:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828700.029053:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828700.029056:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029059:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828700.029060:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828700.029065:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828700.029072:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828700.029078:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828700.029085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327270400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828700.029097:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828700.029102:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828700.029108:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327270400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828700.029123:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828700.029128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828700.029131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828700.029135:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828700.029136:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828700.029138:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828700.029143:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2847
00000400:00000200:2.0:1787828700.029146:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828700.029151:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828700.029154:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828700.029161:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029163:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.029168:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029173:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828700.029178:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828700.029181:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828700.029184:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828700.029188:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828700.029193:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828700.029196:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828700.029203:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029206:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.029210:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029215:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828700.029224:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828700.029229:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029231:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828700.029234:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828700.029237:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828700.029241:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029243:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828700.029246:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828700.029250:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029253:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828700.029259:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828700.029263:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828700.029268:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828700.029271:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029276:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828700.029279:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828700.029281:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828700.029287:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828700.029293:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828700.029299:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327270528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828700.029314:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828700.029318:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828700.029325:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327270528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828700.029336:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828700.029338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828700.029344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828700.029345:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828700.029346:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828700.029351:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828700.029356:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2847
00000400:00000200:2.0:1787828700.029361:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828700.029388:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:0.1:1787828700.029392:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787828700.029395:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1787828700.029401:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787828700.029406:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029411:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.029421:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029426:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828700.029431:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828700.029433:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828700.029436:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828700.029444:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828700.029451:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828700.029459:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828700.029468:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029473:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828700.029512:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029518:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828700.029524:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828700.029531:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828700.029537:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828700.029542:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828700.029565:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828700.029569:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029575:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828700.029579:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828700.029587:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029592:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828700.029600:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828700.029608:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828700.029630:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828700.029633:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828700.029638:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828700.029639:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828700.029642:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828700.029648:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828700.029654:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828700.029664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327270656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828700.029672:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828700.029676:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828700.029682:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327270656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828700.029719:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828700.029728:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2857)
00000800:00000200:0.1:1787828700.029987:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828700.029989:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828700.030223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828700.030226:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828700.030479:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828700.030500:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787828700.540757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828700.540763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828700.540765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828700.540769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828700.540773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828700.540776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828700.540778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828700.540780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828700.540782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828700.732742:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297520812, at 4297520832
00000100:00000010:2.0:1787828700.732761:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000040:3.0:1787828700.732811:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297520812, at 4297520832
00000100:00000010:3.0:1787828700.732826:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000020:00000001:2.0:1787828700.860895:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828700.860900:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828700.860905:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828701.564766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828701.564771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828701.564773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828701.564777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828701.564780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828701.564783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828701.564785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828701.564788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828701.564789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828701.884887:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828701.884892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828701.884897:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828702.588793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828702.588799:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828702.588801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828702.588806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828702.588810:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828702.588814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828702.588818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828702.588822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828702.588826:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828702.908973:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828702.908982:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828702.908990:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828703.612786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828703.612794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828703.612796:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828703.612802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828703.612806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828703.612810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828703.612813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828703.612816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828703.612818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828703.932842:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828703.932846:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828703.932860:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828704.636724:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828704.636731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828704.636732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828704.636736:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828704.636739:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828704.636741:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828704.636743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828704.636744:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828704.636746:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828704.956820:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828704.956823:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828704.956826:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828705.149078:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828705.149083:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828705.149087:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828705.149096:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149122:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828705.149126:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.149129:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828705.149133:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828705.149137:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828705.149143:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327270784:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828705.149150:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828705.149154:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828705.149158:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.149160:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828705.149169:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828705.149175:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828705.149195:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828705.149204:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828705.149212:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828705.149222:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327270784, portal 25
00000100:00000001:3.0:1787828705.149227:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828705.149230:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:3.0:1787828705.149247:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149260:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828705.149266:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828705.149271:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828705.149275:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327270784, offset 0
00000400:00000200:3.0:1787828705.149284:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828705.149296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.149302:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828705.149309:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828705.149316:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828705.149322:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828705.149328:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828705.149334:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39887:39887:256:4294967295] 192.168.203.116@tcp LPNI seq info [39887:39887:8:4294967295]
00000400:00000200:3.0:1787828705.149340:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.149352:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828705.149361:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.149366:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828705.149375:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828705.149380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828705.149400:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828705.149421:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828705.149432:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.149437:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828705.149458:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828705.149468:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828705.149475:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828705.149479:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828705.149486:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828705.149498:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149520:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.149527:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149544:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.149549:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828705.149559:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828705.149566:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828705.149571:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297525932
00000100:00000010:3.0:1787828705.149578:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fae0.
00000100:00000001:3.0:1787828705.149587:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828705.149594:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297525932
00000100:00000010:3.0:1787828705.149598:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f840.
00000100:00000001:3.0:1787828705.149603:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828705.149608:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828705.149613:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.149618:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828705.149624:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828705/real 1787828705]  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828705.149649:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828705.149653:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828705.149686:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828705.149689:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828705.149693:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828705.149697:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828705.149702:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828705.149716:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149726:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828705.149727:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.149729:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.149733:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828705.149746:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149774:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828705.149781:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149791:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828705.149794:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.149799:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.149801:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.149805:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828705.149807:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828705.149812:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828705.149817:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828705.149822:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828705.149826:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828705.149832:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327270784:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828705.149836:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.149838:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.149840:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327270784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149844:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828705.149846:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828705.149848:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828705.149850:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828705.149851:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828705.149853:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828705.149854:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828705.149856:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:3.0:1787828705.149859:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828705.149860:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828705.149861:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:3.0:1787828705.149863:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828705.149864:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.149865:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.149871:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828705.149872:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.149874:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828705.149875:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828705.149876:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828705.149879:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149884:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828705.149885:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.149886:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828705.149888:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828705.149889:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828705.149891:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327270912:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828705.149893:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828705.149895:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828705.149896:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.149897:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828705.149900:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828705.149906:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828705.149909:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828705.149911:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828705.149914:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828705.149916:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327270912, portal 10
00000100:00000001:3.0:1787828705.149918:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828705.149918:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:3.0:1787828705.149923:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.149928:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828705.149930:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828705.149931:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828705.149933:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327270912, offset 0
00000400:00000200:3.0:1787828705.149935:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828705.149938:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.149941:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828705.149943:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828705.149945:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828705.149947:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828705.149949:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828705.149951:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39888:39888:256:4294967295] 192.168.203.116@tcp LPNI seq info [39888:39888:8:4294967295]
00000400:00000200:3.0:1787828705.149954:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.149959:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828705.149962:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.149964:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828705.149967:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828705.149969:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828705.149972:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828705.149974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828705.149977:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.149978:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828705.149981:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828705.149983:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828705.149985:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828705.150039:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828705.150042:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828705.150045:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150051:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.150053:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150057:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150058:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828705.150061:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828705.150064:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828705.150066:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828705.150068:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297525259
00000100:00000001:3.0:1787828705.150070:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297525932
00000100:00000001:3.0:1787828705.150072:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297525259
00000100:00000001:3.0:1787828705.150073:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297525932
00000100:00000001:3.0:1787828705.150075:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828705.150077:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150078:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828705.150080:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828705/real 1787828705]  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828705.150085:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828705.150086:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828705.150088:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828705.150089:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828705.150090:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828705.150091:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828705.150093:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828705.150098:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150103:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828705.150103:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150104:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.150105:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828705.150108:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150119:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828705.150121:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150125:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828705.150127:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150128:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.150128:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150130:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828705.150131:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828705.150133:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828705.150135:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828705.150137:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828705.150139:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828705.150144:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327270912:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828705.150146:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828705.150148:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150152:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828705.150153:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.150154:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828705.150156:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828705.150157:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828705.150159:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271040:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828705.150160:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828705.150162:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828705.150163:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.150165:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828705.150168:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828705.150171:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828705.150174:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828705.150177:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828705.150181:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828705.150184:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271040, portal 4
00000100:00000001:3.0:1787828705.150197:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828705.150198:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:3.0:1787828705.150204:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150217:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828705.150220:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828705.150222:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828705.150225:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327271040, offset 0
00000400:00000200:3.0:1787828705.150230:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828705.150234:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.150238:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828705.150241:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828705.150244:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828705.150246:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828705.150249:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828705.150252:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39889:39889:256:4294967295] 192.168.203.116@tcp LPNI seq info [39889:39889:8:4294967295]
00000400:00000200:3.0:1787828705.150257:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.150264:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828705.150269:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.150273:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828705.150277:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828705.150279:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828705.150282:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828705.150284:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828705.150287:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.150288:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828705.150291:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828705.150293:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828705.150295:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828705.150297:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828705.150299:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828705.150301:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150307:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.150309:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150318:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150319:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828705.150321:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828705.150323:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828705.150326:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828705.150327:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297525266
00000100:00000001:3.0:1787828705.150329:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297525932
00000100:00000001:3.0:1787828705.150330:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297525266
00000100:00000001:3.0:1787828705.150332:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297525932
00000100:00000001:3.0:1787828705.150333:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828705.150334:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150336:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828705.150338:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828705/real 1787828705]  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828705.150343:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828705.150344:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828705.150345:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828705.150347:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828705.150348:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828705.150349:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828705.150351:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828705.150356:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150360:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828705.150361:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150362:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.150363:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828705.150365:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150370:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828705.150373:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150377:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828705.150378:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150379:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.150380:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150381:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828705.150382:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828705.150384:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828705.150386:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828705.150387:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828705.150389:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828705.150394:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271040:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828705.150397:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828705.150399:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150403:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828705.150404:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.150405:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828705.150406:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828705.150408:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828705.150410:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828705.150412:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828705.150413:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828705.150414:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828705.150416:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828705.150418:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828705.150419:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828705.150421:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828705.150423:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828705.150425:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828705.150427:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271168, portal 4
00000100:00000001:3.0:1787828705.150428:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828705.150429:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:3.0:1787828705.150433:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150437:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828705.150438:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828705.150440:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828705.150441:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327271168, offset 0
00000400:00000200:3.0:1787828705.150460:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828705.150463:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.150466:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828705.150468:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828705.150469:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828705.150471:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828705.150473:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828705.150475:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39890:39890:256:4294967295] 192.168.203.116@tcp LPNI seq info [39890:39890:8:4294967295]
00000400:00000200:3.0:1787828705.150477:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828705.150482:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828705.150485:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.150487:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828705.150490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828705.150491:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828705.150494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828705.150496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828705.150499:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828705.150501:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828705.150503:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828705.150505:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828705.150508:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828705.150510:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828705.150513:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828705.150516:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150524:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.150527:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150534:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150536:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828705.150539:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828705.150542:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828705.150546:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828705.150548:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297525271
00000100:00000001:3.0:1787828705.150551:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297525932
00000100:00000001:3.0:1787828705.150553:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297525271
00000100:00000001:3.0:1787828705.150555:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297525932
00000100:00000001:3.0:1787828705.150558:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828705.150560:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150562:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828705.150566:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828705/real 1787828705]  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828705.150576:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828705.150577:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828705.150579:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828705.150582:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828705.150583:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828705.150586:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828705.150588:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828705.150594:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150598:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828705.150599:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150600:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.150601:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828705.150603:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150608:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828705.150610:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150615:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828705.150616:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150617:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828705.150618:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150619:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828705.150620:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828705.150622:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828705.150624:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828705.150625:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828705.150628:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828705.150637:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828705.150640:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150642:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.150646:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327270912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150653:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828705.150675:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828705.150677:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828705.150680:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828705.150681:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828705.150682:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828705.150684:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828705.150685:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:3.0:1787828705.150687:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828705.150688:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828705.150689:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:3.0:1787828705.150691:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828705.150692:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150694:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.150696:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327271040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150701:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828705.150702:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828705.150703:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828705.150705:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828705.150705:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828705.150706:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828705.150707:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828705.150708:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:3.0:1787828705.150709:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828705.150710:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828705.150711:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:3.0:1787828705.150720:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828705.150721:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150722:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828705.150723:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327271168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828705.150727:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828705.150728:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828705.150729:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828705.150731:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828705.150732:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828705.150733:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828705.150734:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828705.150735:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:3.0:1787828705.150736:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828705.150736:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828705.150737:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:3.0:1787828705.150739:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828705.150739:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150740:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828705.150743:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828705.150744:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828705.150745:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828705.150746:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828705.660796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828705.660803:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828705.660805:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828705.660810:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828705.660814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828705.660817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828705.660821:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828705.660824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828705.660827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828705.852720:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297525932, at 4297525952
00000100:00000010:3.0:1787828705.852739:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f840.
00000100:00000040:2.0:1787828705.852771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297525932, at 4297525952
00000100:00000010:2.0:1787828705.852785:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fae0.
00000020:00000001:3.0:1787828705.980842:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828705.980845:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828705.980848:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828706.684732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828706.684739:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828706.684742:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828706.684748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828706.684752:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828706.684755:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828706.684758:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828706.684760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828706.684764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828707.004848:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828707.004852:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828707.004854:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828707.708739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828707.708744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828707.708746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828707.708751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828707.708754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828707.708757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828707.708759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828707.708761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828707.708763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828708.732790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828708.732797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828708.732800:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828708.732806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828708.732810:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828708.732814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828708.732825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828708.732828:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828708.732832:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828709.052997:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828709.053003:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828709.053009:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828709.756754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828709.756762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828709.756765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828709.756770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828709.756791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828709.756794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828709.756796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828709.756799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828709.756801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828710.076823:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828710.076828:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828710.076831:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828710.269285:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828710.269306:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828710.269312:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828710.269329:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.269359:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828710.269364:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.269371:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828710.269378:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828710.269387:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828710.269396:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327271296:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828710.269427:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828710.269442:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828710.269467:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.269477:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828710.269490:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828710.269498:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828710.269513:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828710.269523:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828710.269538:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828710.269551:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271296, portal 25
00000100:00000001:3.0:1787828710.269560:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828710.269565:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:3.0:1787828710.269591:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.269613:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828710.269623:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828710.269631:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828710.269640:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327271296, offset 0
00000400:00000200:3.0:1787828710.269662:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828710.269722:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.269738:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828710.269753:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828710.269766:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828710.269777:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828710.269786:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828710.269799:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39891:39891:256:4294967295] 192.168.203.116@tcp LPNI seq info [39891:39891:8:4294967295]
00000400:00000200:3.0:1787828710.269816:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.269844:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828710.269864:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.269893:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828710.269910:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828710.269922:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828710.269935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828710.269943:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828710.269956:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.269962:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828710.269976:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828710.269991:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828710.270003:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828710.270009:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828710.270018:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828710.270029:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270058:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.270068:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270097:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.270104:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828710.270117:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828710.270128:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828710.270136:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297531053
00000100:00000010:3.0:1787828710.270148:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f840.
00000100:00000001:3.0:1787828710.270163:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828710.270188:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297531053
00000100:00000010:3.0:1787828710.270197:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f300.
00000100:00000001:3.0:1787828710.270207:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828710.270220:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828710.270231:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.270239:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828710.270252:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828710/real 1787828710]  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828710.270291:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828710.270295:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828710.270301:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828710.270309:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828710.270315:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828710.270324:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828710.270335:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828710.270358:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270376:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828710.270381:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.270388:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.270393:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828710.270406:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270496:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828710.270537:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270550:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828710.270555:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.270558:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.270560:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.270564:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828710.270567:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828710.270600:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828710.270608:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828710.270612:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828710.270619:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828710.270632:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327271296:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828710.270643:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.270647:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.270651:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327271296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270662:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828710.270666:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828710.270670:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828710.270675:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828710.270678:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828710.270681:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828710.270685:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828710.270689:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:3.0:1787828710.270696:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828710.270699:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828710.270701:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:3.0:1787828710.270739:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828710.270741:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.270745:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.270756:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828710.270760:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.270763:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828710.270766:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828710.270768:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828710.270774:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270786:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828710.270789:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.270792:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828710.270795:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828710.270798:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828710.270803:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271424:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828710.270808:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828710.270811:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828710.270813:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.270817:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828710.270822:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828710.270825:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828710.270831:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828710.270836:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828710.270842:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828710.270846:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271424, portal 10
00000100:00000001:3.0:1787828710.270849:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828710.270851:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:3.0:1787828710.270862:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.270871:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828710.270874:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828710.270877:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828710.270881:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327271424, offset 0
00000400:00000200:3.0:1787828710.270887:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828710.270908:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.270913:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828710.270918:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828710.270922:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828710.270925:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828710.270928:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828710.270933:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39892:39892:256:4294967295] 192.168.203.116@tcp LPNI seq info [39892:39892:8:4294967295]
00000400:00000200:3.0:1787828710.270939:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.270947:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828710.270954:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.270959:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828710.270965:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828710.270969:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828710.270975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828710.270979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828710.270985:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.270987:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828710.270993:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828710.270998:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828710.271002:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828710.271005:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828710.271008:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828710.271013:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271024:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.271029:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271038:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.271041:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828710.271046:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828710.271051:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828710.271056:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828710.271059:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297530380
00000100:00000001:3.0:1787828710.271063:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297531053
00000100:00000001:3.0:1787828710.271066:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297530380
00000100:00000001:3.0:1787828710.271069:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297531053
00000100:00000001:3.0:1787828710.271072:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828710.271075:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.271077:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828710.271083:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828710/real 1787828710]  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828710.271095:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828710.271097:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828710.271100:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828710.271103:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828710.271106:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828710.271109:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828710.271114:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828710.271126:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271136:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828710.271138:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.271141:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.271143:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828710.271149:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271159:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828710.271165:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271175:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828710.271179:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.271182:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.271183:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.271186:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828710.271189:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828710.271193:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828710.271198:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828710.271201:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828710.271207:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828710.271219:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271424:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828710.271225:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828710.271230:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271252:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828710.271254:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.271257:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828710.271260:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828710.271262:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828710.271266:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271552:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828710.271271:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828710.271274:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828710.271276:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.271279:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828710.271284:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828710.271286:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828710.271291:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828710.271295:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828710.271299:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828710.271303:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271552, portal 4
00000100:00000001:3.0:1787828710.271305:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828710.271307:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:3.0:1787828710.271315:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271323:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828710.271326:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828710.271329:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828710.271332:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327271552, offset 0
00000400:00000200:3.0:1787828710.271337:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828710.271343:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.271348:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828710.271352:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828710.271356:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828710.271359:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828710.271362:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828710.271367:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39893:39893:256:4294967295] 192.168.203.116@tcp LPNI seq info [39893:39893:8:4294967295]
00000400:00000200:3.0:1787828710.271373:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.271381:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828710.271388:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.271393:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828710.271399:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828710.271403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828710.271427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828710.271431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828710.271437:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.271439:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828710.271444:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828710.271449:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828710.271453:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828710.271456:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828710.271459:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828710.271464:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271476:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.271480:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271490:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.271493:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828710.271497:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828710.271501:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828710.271506:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828710.271509:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297530387
00000100:00000001:3.0:1787828710.271513:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297531053
00000100:00000001:3.0:1787828710.271515:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297530387
00000100:00000001:3.0:1787828710.271518:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297531053
00000100:00000001:3.0:1787828710.271538:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828710.271542:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.271545:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828710.271552:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828710/real 1787828710]  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828710.271566:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828710.271568:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828710.271571:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828710.271575:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828710.271606:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828710.271609:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828710.271615:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828710.271630:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271640:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828710.271642:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.271645:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.271648:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828710.271654:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271664:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828710.271670:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271683:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828710.271714:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.271718:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.271720:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.271723:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828710.271726:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828710.271730:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828710.271737:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828710.271741:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828710.271749:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828710.271761:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271552:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828710.271767:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828710.271773:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271783:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828710.271786:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.271790:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828710.271793:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828710.271797:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828710.271801:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271680:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828710.271807:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828710.271809:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828710.271812:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828710.271815:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828710.271821:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828710.271824:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828710.271829:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828710.271833:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828710.271847:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828710.271852:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271680, portal 4
00000100:00000001:3.0:1787828710.271855:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828710.271857:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:3.0:1787828710.271866:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.271876:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828710.271881:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828710.271884:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828710.271905:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327271680, offset 0
00000400:00000200:3.0:1787828710.271910:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828710.271916:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.271921:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828710.271925:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828710.271929:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828710.271932:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828710.271935:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828710.271939:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39894:39894:256:4294967295] 192.168.203.116@tcp LPNI seq info [39894:39894:8:4294967295]
00000400:00000200:3.0:1787828710.271945:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828710.271953:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828710.271960:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.271964:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828710.271970:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828710.271974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828710.271980:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828710.271984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828710.271990:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828710.271992:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828710.271997:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828710.272002:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828710.272005:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828710.272008:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828710.272012:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828710.272016:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272027:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.272032:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272041:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.272043:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828710.272048:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828710.272052:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828710.272057:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828710.272060:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297530393
00000100:00000001:3.0:1787828710.272063:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297531053
00000100:00000001:3.0:1787828710.272066:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297530393
00000100:00000001:3.0:1787828710.272069:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297531053
00000100:00000001:3.0:1787828710.272072:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828710.272075:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.272078:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828710.272083:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828710/real 1787828710]  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828710.272094:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828710.272095:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828710.272098:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828710.272101:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828710.272103:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828710.272106:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828710.272109:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828710.272119:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272127:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828710.272129:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272131:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.272132:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828710.272136:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272145:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828710.272150:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272161:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828710.272165:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272167:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828710.272169:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272171:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828710.272173:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828710.272176:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828710.272180:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828710.272193:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828710.272197:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828710.272206:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271680:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828710.272210:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272213:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.272216:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327271424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272224:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828710.272226:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828710.272229:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828710.272232:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828710.272233:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828710.272235:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828710.272238:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828710.272240:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:3.0:1787828710.272243:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828710.272245:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828710.272247:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:3.0:1787828710.272251:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828710.272253:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272255:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.272259:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327271552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272267:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828710.272268:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828710.272270:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828710.272282:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828710.272284:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828710.272286:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828710.272288:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828710.272290:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:3.0:1787828710.272293:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828710.272294:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828710.272296:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:3.0:1787828710.272298:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828710.272300:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272303:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828710.272307:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327271680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828710.272315:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828710.272317:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828710.272319:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828710.272321:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828710.272322:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828710.272324:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828710.272326:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828710.272328:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:3.0:1787828710.272330:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828710.272332:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828710.272333:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:3.0:1787828710.272336:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828710.272337:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272339:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828710.272343:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828710.272345:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828710.272347:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828710.272350:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828710.780770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828710.780777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828710.780779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828710.780784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828710.780787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828710.780790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828710.780793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828710.780795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828710.780798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828710.972744:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297531053, at 4297531072
00000100:00000040:2.0:1787828710.972748:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297531053, at 4297531072
00000100:00000010:3.0:1787828710.972778:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f300.
00000100:00000010:2.0:1787828710.972784:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f840.
00000100:00000001:2.0:1787828711.804764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828711.804771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828711.804773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828711.804778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828711.804781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828711.804786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828711.804788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828711.804791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828711.804794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828712.124828:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828712.124832:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828712.124835:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828712.828765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828712.828772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828712.828774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828712.828780:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828712.828783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828712.828787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828712.828789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828712.828792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828712.828795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828713.148859:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828713.148862:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828713.148866:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828713.852795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828713.852803:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828713.852805:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828713.852811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828713.852814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828713.852818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828713.852821:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828713.852823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828713.852826:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828715.196839:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828715.196845:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828715.196848:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828715.388991:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828715.388997:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828715.389000:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828715.389009:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389025:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828715.389028:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.389034:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828715.389038:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828715.389041:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828715.389047:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327271808:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828715.389053:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828715.389057:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828715.389060:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.389063:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828715.389070:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828715.389074:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828715.389081:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828715.389087:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828715.389095:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828715.389105:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271808, portal 25
00000100:00000001:3.0:1787828715.389120:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828715.389124:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:3.0:1787828715.389139:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389166:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828715.389173:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828715.389179:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828715.389186:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327271808, offset 0
00000400:00000200:3.0:1787828715.389204:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828715.389215:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.389225:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828715.389239:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828715.389245:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828715.389252:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828715.389258:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828715.389264:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39895:39895:256:4294967295] 192.168.203.116@tcp LPNI seq info [39895:39895:8:4294967295]
00000400:00000200:3.0:1787828715.389270:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.389284:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828715.389293:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.389301:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828715.389312:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828715.389317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828715.389327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828715.389332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828715.389339:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.389343:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828715.389352:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828715.389361:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828715.389368:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828715.389385:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828715.389391:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828715.389399:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389418:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.389424:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389447:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.389450:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828715.389456:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828715.389463:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828715.389468:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297536172
00000100:00000010:3.0:1787828715.389475:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f300.
00000100:00000001:3.0:1787828715.389483:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828715.389496:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297536172
00000100:00000010:3.0:1787828715.389502:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fde0.
00000100:00000001:3.0:1787828715.389507:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828715.389512:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828715.389515:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.389518:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828715.389525:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715]  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828715.389532:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828715.389535:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828715.389537:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828715.389540:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828715.389542:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828715.389544:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828715.389546:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828715.389552:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389556:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828715.389557:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389559:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.389560:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828715.389562:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389568:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828715.389571:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389575:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828715.389577:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389578:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.389579:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389581:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828715.389582:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828715.389584:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828715.389586:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828715.389588:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828715.389590:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828715.389595:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327271808:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828715.389599:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389600:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.389602:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327271808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389606:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828715.389608:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828715.389610:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828715.389613:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828715.389614:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828715.389615:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828715.389617:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828715.389618:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:3.0:1787828715.389621:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828715.389623:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828715.389624:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:3.0:1787828715.389626:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828715.389627:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389628:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389633:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828715.389635:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.389636:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828715.389637:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828715.389638:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828715.389641:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389645:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828715.389646:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.389648:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828715.389649:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828715.389651:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828715.389687:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271936:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828715.389691:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828715.389694:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828715.389696:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.389698:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828715.389715:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828715.389718:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828715.389722:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828715.389725:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828715.389729:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828715.389732:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327271936, portal 10
00000100:00000001:3.0:1787828715.389734:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828715.389736:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:3.0:1787828715.389743:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389750:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828715.389753:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828715.389756:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828715.389758:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327271936, offset 0
00000400:00000200:3.0:1787828715.389763:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828715.389768:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.389772:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828715.389784:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828715.389787:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828715.389790:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828715.389792:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828715.389795:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39896:39896:256:4294967295] 192.168.203.116@tcp LPNI seq info [39896:39896:8:4294967295]
00000400:00000200:3.0:1787828715.389800:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.389807:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828715.389812:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.389816:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828715.389820:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828715.389823:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828715.389828:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828715.389831:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828715.389836:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.389838:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828715.389842:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828715.389845:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828715.389849:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828715.389851:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828715.389854:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828715.389858:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389866:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.389870:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389877:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.389878:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828715.389883:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828715.389887:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828715.389891:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828715.389893:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297535499
00000100:00000001:3.0:1787828715.389896:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297536172
00000100:00000001:3.0:1787828715.389898:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297535499
00000100:00000001:3.0:1787828715.389900:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297536172
00000100:00000001:3.0:1787828715.389902:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828715.389904:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.389906:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828715.389910:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715]  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828715.389919:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828715.389921:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828715.389923:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828715.389925:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828715.389927:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828715.389929:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828715.389933:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828715.389942:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389950:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828715.389951:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389953:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.389954:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828715.389959:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389966:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828715.389971:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.389979:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828715.389981:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389983:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.389984:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.389986:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828715.389989:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828715.389992:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828715.389995:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828715.389998:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828715.390002:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828715.390011:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327271936:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828715.390015:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828715.390019:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390027:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828715.390028:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.390031:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828715.390032:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828715.390035:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828715.390038:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272064:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828715.390041:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828715.390043:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828715.390045:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.390047:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828715.390051:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828715.390053:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828715.390056:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828715.390060:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828715.390063:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828715.390066:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272064, portal 4
00000100:00000001:3.0:1787828715.390068:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828715.390070:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:3.0:1787828715.390076:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390083:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828715.390085:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828715.390088:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828715.390091:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327272064, offset 0
00000400:00000200:3.0:1787828715.390095:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828715.390100:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.390103:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828715.390107:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828715.390110:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828715.390112:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828715.390115:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828715.390118:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39897:39897:256:4294967295] 192.168.203.116@tcp LPNI seq info [39897:39897:8:4294967295]
00000400:00000200:3.0:1787828715.390123:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.390129:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828715.390134:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.390138:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828715.390142:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828715.390145:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828715.390150:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828715.390153:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828715.390158:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.390160:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828715.390164:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828715.390168:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828715.390171:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828715.390173:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828715.390176:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828715.390180:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390189:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.390192:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390199:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.390201:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828715.390204:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828715.390208:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828715.390211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828715.390213:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297535506
00000100:00000001:3.0:1787828715.390216:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297536172
00000100:00000001:3.0:1787828715.390219:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297535506
00000100:00000001:3.0:1787828715.390221:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297536172
00000100:00000001:3.0:1787828715.390224:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828715.390226:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.390228:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828715.390232:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715]  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828715.390241:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828715.390243:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828715.390245:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828715.390248:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828715.390250:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828715.390252:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828715.390256:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828715.390265:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390278:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828715.390280:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390282:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.390283:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828715.390287:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390295:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828715.390299:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390307:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828715.390310:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390311:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.390313:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390315:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828715.390317:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828715.390319:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828715.390323:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828715.390325:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828715.390330:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828715.390339:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272064:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828715.390343:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828715.390347:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390354:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828715.390356:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.390359:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828715.390361:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828715.390363:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828715.390366:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272192:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828715.390369:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828715.390372:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828715.390388:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828715.390390:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828715.390395:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828715.390397:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828715.390400:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828715.390404:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828715.390408:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828715.390411:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272192, portal 4
00000100:00000001:3.0:1787828715.390413:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828715.390414:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:3.0:1787828715.390421:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390427:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828715.390430:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828715.390432:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828715.390435:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327272192, offset 0
00000400:00000200:3.0:1787828715.390439:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828715.390443:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.390447:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828715.390450:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828715.390453:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828715.390455:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828715.390458:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828715.390461:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39898:39898:256:4294967295] 192.168.203.116@tcp LPNI seq info [39898:39898:8:4294967295]
00000400:00000200:3.0:1787828715.390465:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828715.390472:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828715.390478:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.390481:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828715.390486:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828715.390489:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828715.390493:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828715.390496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828715.390501:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828715.390503:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828715.390507:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828715.390510:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828715.390513:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828715.390515:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828715.390518:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828715.390522:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390531:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.390534:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390541:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.390543:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828715.390547:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828715.390550:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828715.390554:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828715.390556:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297535511
00000100:00000001:3.0:1787828715.390559:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297536172
00000100:00000001:3.0:1787828715.390561:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297535511
00000100:00000001:3.0:1787828715.390563:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297536172
00000100:00000001:3.0:1787828715.390566:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828715.390568:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.390570:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828715.390575:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715]  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828715.390584:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828715.390586:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828715.390587:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828715.390590:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828715.390592:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828715.390594:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828715.390598:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828715.390607:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390614:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828715.390615:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390617:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.390619:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828715.390623:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390630:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828715.390635:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390642:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828715.390645:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390647:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828715.390648:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390650:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828715.390652:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828715.390676:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828715.390680:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828715.390682:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828715.390687:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828715.390697:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272192:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828715.390700:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390702:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.390706:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327271936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390713:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828715.390715:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828715.390717:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828715.390720:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828715.390721:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828715.390722:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828715.390724:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828715.390726:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:3.0:1787828715.390729:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828715.390730:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828715.390732:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:3.0:1787828715.390735:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828715.390736:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390738:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.390742:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327272064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390749:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828715.390750:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828715.390752:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828715.390755:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828715.390756:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828715.390757:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828715.390759:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828715.390761:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:3.0:1787828715.390763:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828715.390765:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828715.390766:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:3.0:1787828715.390769:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828715.390771:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390773:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828715.390776:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327272192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828715.390783:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828715.390785:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828715.390786:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828715.390789:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828715.390790:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828715.390791:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828715.390793:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828715.390795:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:3.0:1787828715.390797:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828715.390798:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828715.390799:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:3.0:1787828715.390802:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828715.390803:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390805:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828715.390809:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828715.390810:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828715.390812:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828715.390814:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828716.092784:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297536172, at 4297536192
00000100:00000040:2.0:1787828716.092790:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297536172, at 4297536192
00000100:00000010:3.0:1787828716.092798:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fde0.
00000100:00000010:2.0:1787828716.092809:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f300.
00000020:00000001:2.0:1787828716.220846:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828716.220851:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828716.220855:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828718.268893:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828718.268898:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828718.268903:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828719.292869:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828719.292875:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828719.292880:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828720.508963:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828720.508970:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828720.508975:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828720.508986:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509009:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828720.509014:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.509020:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828720.509024:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828720.509029:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828720.509036:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327272320:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828720.509046:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828720.509052:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828720.509057:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.509063:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828720.509070:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828720.509076:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828720.509091:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828720.509098:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828720.509109:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828720.509119:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272320, portal 25
00000100:00000001:3.0:1787828720.509125:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828720.509129:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:3.0:1787828720.509144:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509167:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828720.509177:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828720.509185:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828720.509190:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327272320, offset 0
00000400:00000200:3.0:1787828720.509198:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828720.509210:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.509217:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828720.509224:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828720.509232:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828720.509239:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828720.509244:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828720.509252:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39899:39899:256:4294967295] 192.168.203.116@tcp LPNI seq info [39899:39899:8:4294967295]
00000400:00000200:3.0:1787828720.509266:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.509285:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828720.509302:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.509312:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828720.509327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828720.509336:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828720.509366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828720.509389:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787828720.509399:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.509403:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828720.509412:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828720.509422:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828720.509432:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828720.509437:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828720.509444:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828720.509450:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509462:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.509465:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509470:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509472:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828720.509477:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828720.509480:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828720.509482:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297541292
00000100:00000010:3.0:1787828720.509486:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fde0.
00000100:00000001:3.0:1787828720.509490:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828720.509494:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297541292
00000100:00000010:3.0:1787828720.509496:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f060.
00000100:00000001:3.0:1787828720.509499:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828720.509501:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828720.509503:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509505:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828720.509507:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828720]  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828720.509519:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828720.509520:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828720.509521:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828720.509523:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828720.509524:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828720.509526:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828720.509528:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828720.509533:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509538:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828720.509539:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509541:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.509542:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828720.509544:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509548:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828720.509551:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509556:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828720.509558:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509559:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.509560:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509561:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828720.509563:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828720.509564:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828720.509567:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828720.509568:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828720.509571:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828720.509577:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327272320:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828720.509581:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509582:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.509584:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327272320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509588:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828720.509590:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828720.509592:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828720.509594:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828720.509595:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828720.509596:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828720.509598:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828720.509600:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:3.0:1787828720.509602:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828720.509603:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828720.509604:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:3.0:1787828720.509607:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828720.509608:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509609:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509614:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828720.509616:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509617:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828720.509619:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828720.509620:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828720.509622:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509627:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828720.509628:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.509629:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828720.509631:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828720.509632:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828720.509634:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272448:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828720.509636:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828720.509638:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828720.509639:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.509640:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828720.509642:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828720.509644:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828720.509646:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828720.509648:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828720.509650:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828720.509652:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272448, portal 10
00000100:00000001:3.0:1787828720.509653:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828720.509654:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:3.0:1787828720.509680:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509685:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828720.509686:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828720.509688:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828720.509690:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327272448, offset 0
00000400:00000200:3.0:1787828720.509693:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828720.509696:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.509699:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828720.509701:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828720.509703:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828720.509705:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828720.509706:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828720.509708:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39900:39900:256:4294967295] 192.168.203.116@tcp LPNI seq info [39900:39900:8:4294967295]
00000400:00000200:3.0:1787828720.509711:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.509715:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828720.509718:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.509720:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828720.509723:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828720.509725:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828720.509728:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828720.509730:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787828720.509733:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.509734:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828720.509737:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828720.509739:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828720.509741:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828720.509742:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828720.509744:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828720.509746:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509752:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.509754:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509758:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509759:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828720.509761:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828720.509764:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828720.509766:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828720.509767:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297540619
00000100:00000001:3.0:1787828720.509769:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297541292
00000100:00000001:3.0:1787828720.509771:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297540619
00000100:00000001:3.0:1787828720.509772:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297541292
00000100:00000001:3.0:1787828720.509774:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828720.509775:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509776:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828720.509779:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828720]  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828720.509783:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828720.509784:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828720.509786:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828720.509787:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828720.509788:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828720.509790:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828720.509792:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828720.509797:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509800:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828720.509801:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509802:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.509803:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828720.509805:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509809:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828720.509812:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509816:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828720.509817:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509819:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.509819:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509821:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828720.509822:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828720.509824:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828720.509826:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828720.509827:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828720.509829:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828720.509834:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272448:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828720.509837:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828720.509839:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509843:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828720.509844:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.509846:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828720.509847:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828720.509848:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828720.509850:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272576:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828720.509852:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828720.509853:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828720.509854:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.509855:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828720.509858:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828720.509859:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828720.509861:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828720.509863:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828720.509865:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828720.509867:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272576, portal 4
00000100:00000001:3.0:1787828720.509868:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828720.509869:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:3.0:1787828720.509873:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509877:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828720.509879:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828720.509880:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828720.509881:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327272576, offset 0
00000400:00000200:3.0:1787828720.509884:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828720.509886:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.509888:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828720.509889:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828720.509891:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828720.509893:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828720.509894:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828720.509896:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39901:39901:256:4294967295] 192.168.203.116@tcp LPNI seq info [39901:39901:8:4294967295]
00000400:00000200:3.0:1787828720.509899:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.509903:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828720.509905:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.509907:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828720.509909:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828720.509911:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828720.509913:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828720.509915:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787828720.509918:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.509919:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828720.509921:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828720.509923:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828720.509925:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828720.509926:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828720.509927:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828720.509930:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509935:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.509936:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509941:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509942:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828720.509943:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828720.509945:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828720.509947:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828720.509948:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297540626
00000100:00000001:3.0:1787828720.509949:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297541292
00000100:00000001:3.0:1787828720.509951:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297540626
00000100:00000001:3.0:1787828720.509952:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297541292
00000100:00000001:3.0:1787828720.509954:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828720.509955:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.509956:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828720.509959:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828720]  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828720.509963:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828720.509964:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828720.509965:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828720.509966:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828720.509967:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828720.509968:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828720.509970:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828720.509974:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509979:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828720.509980:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509981:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.509982:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828720.509984:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509988:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828720.509990:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.509994:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828720.509995:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509996:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.509997:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.509999:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828720.510000:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828720.510002:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828720.510004:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828720.510005:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828720.510007:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828720.510012:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272576:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828720.510014:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828720.510016:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510020:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828720.510021:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.510022:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828720.510023:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828720.510024:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828720.510026:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272704:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828720.510027:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828720.510028:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828720.510029:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828720.510031:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828720.510033:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828720.510034:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828720.510036:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828720.510037:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828720.510039:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828720.510041:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272704, portal 4
00000100:00000001:3.0:1787828720.510042:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828720.510043:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:3.0:1787828720.510046:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510050:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828720.510052:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828720.510053:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828720.510055:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327272704, offset 0
00000400:00000200:3.0:1787828720.510057:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828720.510059:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.510061:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828720.510062:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828720.510064:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828720.510065:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828720.510067:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828720.510069:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39902:39902:256:4294967295] 192.168.203.116@tcp LPNI seq info [39902:39902:8:4294967295]
00000400:00000200:3.0:1787828720.510071:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828720.510075:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828720.510078:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.510079:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828720.510082:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828720.510083:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787828720.510086:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828720.510088:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787828720.510090:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828720.510091:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828720.510093:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828720.510095:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828720.510097:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828720.510098:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828720.510099:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828720.510101:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510106:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.510108:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510116:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.510117:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828720.510119:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828720.510120:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828720.510122:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828720.510124:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297540631
00000100:00000001:3.0:1787828720.510125:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297541292
00000100:00000001:3.0:1787828720.510127:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297540631
00000100:00000001:3.0:1787828720.510130:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297541292
00000100:00000001:3.0:1787828720.510132:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828720.510134:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.510135:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828720.510139:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828720]  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828720.510147:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828720.510149:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828720.510150:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828720.510153:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828720.510154:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828720.510157:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828720.510160:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828720.510169:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510177:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828720.510178:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510180:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.510182:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828720.510186:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510192:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828720.510195:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510199:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828720.510201:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510201:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828720.510202:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510203:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828720.510205:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828720.510206:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828720.510208:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828720.510209:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828720.510212:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828720.510217:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272704:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828720.510219:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510220:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.510222:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327272448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510226:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828720.510227:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828720.510228:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828720.510230:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828720.510231:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828720.510232:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828720.510233:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828720.510234:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4800.
02000000:00000001:3.0:1787828720.510236:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828720.510237:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828720.510238:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:3.0:1787828720.510240:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828720.510241:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510242:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.510244:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327272576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510248:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828720.510249:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828720.510250:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828720.510251:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828720.510252:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828720.510253:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828720.510254:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828720.510255:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:3.0:1787828720.510256:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828720.510257:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828720.510257:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:3.0:1787828720.510259:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828720.510260:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510261:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828720.510263:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327272704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828720.510266:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828720.510267:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828720.510268:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828720.510269:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828720.510270:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828720.510270:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828720.510271:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828720.510272:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:3.0:1787828720.510273:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828720.510274:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828720.510275:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:3.0:1787828720.510276:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828720.510277:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510278:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828720.510281:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828720.510282:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828720.510283:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828720.510284:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828721.212766:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297541292, at 4297541312
00000100:00000040:3.0:1787828721.212766:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297541292, at 4297541312
00000100:00000010:2.0:1787828721.212780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fde0.
00000100:00000010:3.0:1787828721.212784:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f060.
00000020:00000001:3.0:1787828721.340880:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828721.340885:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828721.340889:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828722.364878:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828722.364886:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828722.364892:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828724.412883:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828724.412887:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828724.412889:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828725.436913:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828725.436916:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828725.436919:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828725.628976:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828725.628982:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828725.628985:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828725.628994:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629010:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828725.629013:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.629018:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828725.629021:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828725.629025:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828725.629030:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327272832:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828725.629036:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828725.629040:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828725.629043:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.629046:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828725.629051:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828725.629055:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828725.629065:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828725.629069:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828725.629077:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828725.629096:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272832, portal 25
00000100:00000001:3.0:1787828725.629100:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828725.629101:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:3.0:1787828725.629106:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629117:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828725.629121:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828725.629126:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828725.629131:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327272832, offset 0
00000400:00000200:3.0:1787828725.629140:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828725.629150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.629158:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828725.629167:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828725.629173:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828725.629178:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828725.629184:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828725.629192:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39903:39903:256:4294967295] 192.168.203.116@tcp LPNI seq info [39903:39903:8:4294967295]
00000400:00000200:3.0:1787828725.629210:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.629225:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828725.629236:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.629243:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828725.629255:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828725.629261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828725.629272:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828725.629277:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828725.629290:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.629296:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828725.629320:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828725.629343:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828725.629351:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828725.629355:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828725.629363:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828725.629368:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629396:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.629402:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629411:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.629415:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828725.629422:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828725.629429:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828725.629434:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297546412
00000100:00000010:3.0:1787828725.629442:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f060.
00000100:00000001:3.0:1787828725.629451:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828725.629466:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297546412
00000100:00000010:3.0:1787828725.629471:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f180.
00000100:00000001:3.0:1787828725.629477:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828725.629481:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828725.629484:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.629490:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828725.629499:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725]  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828725.629515:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828725.629518:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828725.629522:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828725.629528:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828725.629532:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828725.629538:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828725.629543:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828725.629561:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629572:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828725.629576:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629580:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.629582:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828725.629588:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629598:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828725.629604:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629624:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828725.629629:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629632:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.629635:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629637:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828725.629640:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828725.629646:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828725.629677:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828725.629680:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828725.629686:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828725.629696:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327272832:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828725.629700:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629702:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.629704:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327272832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629710:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828725.629712:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828725.629715:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828725.629718:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828725.629720:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828725.629721:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828725.629723:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828725.629725:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:3.0:1787828725.629727:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828725.629729:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828725.629730:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:3.0:1787828725.629733:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828725.629734:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629735:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629747:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828725.629748:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.629750:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828725.629751:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828725.629752:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828725.629755:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629760:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828725.629761:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.629762:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828725.629764:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828725.629765:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828725.629767:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272960:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828725.629770:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828725.629771:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828725.629772:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.629774:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828725.629776:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828725.629777:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828725.629780:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828725.629781:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828725.629784:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828725.629786:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327272960, portal 10
00000100:00000001:3.0:1787828725.629787:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828725.629789:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:3.0:1787828725.629793:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629798:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828725.629800:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828725.629801:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828725.629803:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327272960, offset 0
00000400:00000200:3.0:1787828725.629806:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828725.629808:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.629811:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828725.629813:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828725.629815:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828725.629816:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828725.629818:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828725.629820:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39904:39904:256:4294967295] 192.168.203.116@tcp LPNI seq info [39904:39904:8:4294967295]
00000400:00000200:3.0:1787828725.629823:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.629828:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828725.629831:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.629833:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828725.629836:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828725.629838:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828725.629841:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828725.629842:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828725.629845:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.629846:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828725.629849:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828725.629851:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828725.629853:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828725.629854:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828725.629856:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828725.629858:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629864:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.629866:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629874:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.629875:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828725.629877:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828725.629885:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828725.629888:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828725.629889:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297545739
00000100:00000001:3.0:1787828725.629891:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297546412
00000100:00000001:3.0:1787828725.629893:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297545739
00000100:00000001:3.0:1787828725.629895:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297546412
00000100:00000001:3.0:1787828725.629896:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828725.629898:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.629899:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828725.629901:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725]  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828725.629906:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828725.629907:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828725.629908:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828725.629909:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828725.629910:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828725.629912:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828725.629914:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828725.629919:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629923:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828725.629924:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629925:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.629926:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828725.629928:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629933:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828725.629935:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629940:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828725.629941:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629942:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.629943:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.629944:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828725.629945:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828725.629947:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828725.629949:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828725.629950:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828725.629952:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828725.629957:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327272960:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828725.629960:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828725.629962:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.629967:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828725.629968:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.629969:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828725.629970:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828725.629971:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828725.629973:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273088:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828725.629975:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828725.629976:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828725.629977:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.629978:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828725.629981:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828725.629982:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828725.629984:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828725.629986:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828725.629988:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828725.629990:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273088, portal 4
00000100:00000001:3.0:1787828725.629991:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828725.629992:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:3.0:1787828725.629995:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630000:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828725.630001:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828725.630003:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828725.630004:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327273088, offset 0
00000400:00000200:3.0:1787828725.630006:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828725.630008:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.630010:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828725.630012:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828725.630014:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828725.630015:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828725.630017:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828725.630019:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39905:39905:256:4294967295] 192.168.203.116@tcp LPNI seq info [39905:39905:8:4294967295]
00000400:00000200:3.0:1787828725.630022:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.630026:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828725.630029:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.630030:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828725.630033:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828725.630034:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828725.630037:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828725.630039:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828725.630041:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.630043:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828725.630045:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828725.630047:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828725.630049:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828725.630050:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828725.630051:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828725.630053:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630059:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.630061:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630065:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.630066:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828725.630067:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828725.630069:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828725.630071:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828725.630072:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297545746
00000100:00000001:3.0:1787828725.630074:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297546412
00000100:00000001:3.0:1787828725.630076:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297545746
00000100:00000001:3.0:1787828725.630077:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297546412
00000100:00000001:3.0:1787828725.630079:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828725.630080:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.630081:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828725.630083:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725]  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828725.630087:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828725.630088:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828725.630089:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828725.630090:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828725.630091:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828725.630092:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828725.630094:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828725.630100:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630104:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828725.630104:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630105:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.630106:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828725.630108:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630112:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828725.630114:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630118:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828725.630120:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630121:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.630122:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630123:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828725.630124:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828725.630125:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828725.630127:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828725.630128:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828725.630131:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828725.630136:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273088:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828725.630138:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828725.630140:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630145:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828725.630146:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.630147:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828725.630149:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828725.630150:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828725.630151:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273216:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828725.630153:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828725.630154:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828725.630155:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828725.630156:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828725.630158:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828725.630159:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828725.630161:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828725.630162:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828725.630164:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828725.630165:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273216, portal 4
00000100:00000001:3.0:1787828725.630166:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828725.630168:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:3.0:1787828725.630171:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630175:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828725.630177:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828725.630178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828725.630179:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327273216, offset 0
00000400:00000200:3.0:1787828725.630182:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828725.630183:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.630185:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828725.630187:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828725.630188:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828725.630190:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828725.630192:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828725.630194:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39906:39906:256:4294967295] 192.168.203.116@tcp LPNI seq info [39906:39906:8:4294967295]
00000400:00000200:3.0:1787828725.630197:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828725.630201:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828725.630204:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.630205:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828725.630208:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828725.630209:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828725.630211:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828725.630213:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828725.630216:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828725.630217:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828725.630219:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828725.630221:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828725.630223:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828725.630224:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828725.630225:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828725.630227:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630232:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.630234:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630238:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.630240:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828725.630241:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828725.630243:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828725.630245:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828725.630246:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297545751
00000100:00000001:3.0:1787828725.630248:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297546412
00000100:00000001:3.0:1787828725.630249:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297545751
00000100:00000001:3.0:1787828725.630251:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297546412
00000100:00000001:3.0:1787828725.630252:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828725.630254:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.630255:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828725.630257:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725]  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828725.630262:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828725.630263:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828725.630264:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828725.630265:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828725.630266:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828725.630267:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828725.630269:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828725.630274:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630279:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828725.630281:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630283:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.630284:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828725.630288:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630296:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828725.630300:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630322:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828725.630325:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630326:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828725.630328:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630330:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828725.630332:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828725.630334:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828725.630337:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828725.630338:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828725.630341:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828725.630346:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273216:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828725.630349:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630350:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.630352:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327272960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630356:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828725.630357:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828725.630358:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828725.630360:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828725.630361:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828725.630362:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828725.630364:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828725.630365:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:3.0:1787828725.630367:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828725.630368:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828725.630369:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:3.0:1787828725.630370:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828725.630371:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630372:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.630374:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327273088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630379:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828725.630380:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828725.630381:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828725.630382:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828725.630383:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828725.630383:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828725.630384:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828725.630386:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:3.0:1787828725.630387:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828725.630388:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828725.630389:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:3.0:1787828725.630390:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828725.630391:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630392:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828725.630394:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327273216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828725.630397:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828725.630398:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828725.630399:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828725.630400:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828725.630401:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828725.630401:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828725.630403:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828725.630403:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:3.0:1787828725.630405:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828725.630405:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828725.630406:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:3.0:1787828725.630407:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828725.630408:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630409:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828725.630411:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828725.630412:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828725.630413:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828725.630414:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828726.332775:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297546412, at 4297546432
00000100:00000040:3.0:1787828726.332779:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297546412, at 4297546432
00000100:00000010:3.0:1787828726.332798:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f180.
00000100:00000010:2.0:1787828726.332805:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f060.
00000020:00000001:3.0:1787828727.484870:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828727.484876:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828727.484881:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828728.508844:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828728.508848:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828728.508852:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828730.556879:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828730.556883:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828730.556887:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828730.749027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828730.749036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828730.749038:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828730.749041:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828730.749044:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828730.749049:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2878
00000400:00000200:2.0:1787828730.749057:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828730.749061:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828730.749064:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828730.749071:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749074:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749075:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749077:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828730.749080:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828730.749082:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828730.749084:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828730.749087:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828730.749088:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828730.749090:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828730.749092:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749093:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749095:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749096:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828730.749098:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828730.749101:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749102:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828730.749104:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828730.749107:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828730.749109:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749110:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828730.749113:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828730.749122:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828730.749126:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828730.749130:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828730.749132:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749135:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828730.749137:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828730.749142:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828730.749150:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828730.749157:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828730.749166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828730.749174:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828730.749177:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828730.749181:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828730.749199:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828730.749202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828730.749204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828730.749205:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828730.749206:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828730.749209:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828730.749210:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2878
00000400:00000200:2.0:1787828730.749213:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828730.749215:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828730.749217:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828730.749219:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749220:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749221:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749223:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828730.749224:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828730.749226:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828730.749228:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828730.749230:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828730.749231:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828730.749233:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828730.749237:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749239:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749241:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749243:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828730.749244:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828730.749247:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749255:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.749256:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787828730.749257:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828730.749262:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828730.749265:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.749280:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828730.749283:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828730.749286:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:2.0:1787828730.749288:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828730.749292:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749294:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828730.749297:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828730.749299:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828730.749303:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828730.749306:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828730.749308:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828730.749312:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828730.749313:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828730.749315:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828730.749322:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828730.749330:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.749340:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00100000:2.0:1787828730.749342:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828730.749345:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749354:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787828730.749356:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787828730.749358:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.749362:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787828730.749363:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787828730.749378:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749382:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828730.749386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828730.749390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00100000:3.0:1787828730.749390:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327273344:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828730.749396:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828730.749399:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828730.749412:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828730.749415:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2878
00000400:00000200:2.0:1787828730.749426:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828730.749427:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000400:00000200:2.0:1787828730.749432:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828730.749435:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1787828730.749437:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1787828730.749438:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749442:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749444:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:3.0:1787828730.749444:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749454:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:3.0:1787828730.749455:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:2.0:1787828730.749457:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828730.749459:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828730.749460:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828730.749462:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828730.749463:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828730.749464:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828730.749466:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749478:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749479:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749480:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828730.749483:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
02000000:00000010:3.0:1787828730.749483:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
00000100:00080000:2.0:1787828730.749492:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:3.0:1787828730.749493:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749498:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828730.749500:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828730.749503:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828730.749505:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749513:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787828730.749514:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:2.0:1787828730.749516:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828730.749523:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749525:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828730.749527:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000100:00000010:2.0:1787828730.749528:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828730.749531:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828730.749544:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828730.749545:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749551:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828730.749553:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828730.749555:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828730.749561:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000400:00000010:3.0:1787828730.749561:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000001:2.0:1787828730.749564:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828730.749572:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787828730.749577:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273344, portal 25
00000100:00000001:2.0:1787828730.749578:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828730.749581:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.749585:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1787828730.749586:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.749590:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787828730.749602:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828730.749605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828730.749612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828730.749613:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1787828730.749614:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828730.749615:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828730.749618:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828730.749631:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2878
00000400:00000200:2.0:1787828730.749639:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828730.749646:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828730.749650:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787828730.749666:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828730.749672:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828730.749677:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000040:2.0:1787828730.749681:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749695:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749697:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749699:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:3.0:1787828730.749701:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327273344, offset 0
00000020:00000001:2.0:1787828730.749702:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828730.749704:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828730.749709:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828730.749710:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828730.749712:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828730.749713:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828730.749722:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828730.749726:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749727:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828730.749731:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749733:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828730.749734:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828730.749736:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828730.749738:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828730.749740:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828730.749742:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828730.749743:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749745:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828730.749746:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828730.749747:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749748:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828730.749751:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828730.749752:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828730.749753:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828730.749754:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828730.749755:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828730.749757:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828730.749758:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828730.749761:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828730.749764:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828730.749769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828730.749774:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828730.749776:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828730.749780:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828730.749785:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828730.749794:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2888)
00000400:00000200:3.0:1787828730.749803:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.749816:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828730.749824:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828730.749830:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828730.749836:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828730.749839:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828730.749844:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39907:39907:256:4294967295] 192.168.203.116@tcp LPNI seq info [39907:39907:8:4294967295]
00000400:00000200:3.0:1787828730.749850:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.749863:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828730.749872:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.749910:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828730.749921:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828730.749925:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828730.749933:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828730.749937:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828730.749942:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.749945:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828730.749956:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828730.749962:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828730.749969:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828730.749972:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828730.749978:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828730.749984:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.749996:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.750001:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750009:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750012:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828730.750020:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828730.750026:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828730.750029:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297551533
00000100:00000010:3.0:1787828730.750036:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f180.
00000100:00000001:3.0:1787828730.750045:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828730.750059:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297551533
00000100:00000010:3.0:1787828730.750064:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f780.
00000100:00000001:3.0:1787828730.750069:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828730.750073:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828730.750077:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750081:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828730.750089:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730]  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828730.750102:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828730.750104:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828730.750106:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828730.750110:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828730.750112:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828730.750115:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828730.750119:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828730.750128:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750136:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828730.750139:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750141:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.750144:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828730.750149:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750158:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828730.750163:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750172:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828730.750176:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750178:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.750180:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750183:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828730.750186:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828730.750190:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828730.750195:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828730.750198:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828730.750203:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828730.750213:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327273344:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828730.750219:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750222:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.750226:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327273344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750234:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828730.750237:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828730.750240:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828730.750244:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828730.750247:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828730.750249:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828730.750252:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828730.750256:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:3.0:1787828730.750261:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828730.750263:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828730.750265:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:3.0:1787828730.750283:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828730.750284:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750287:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750297:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828730.750299:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750302:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828730.750304:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828730.750306:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828730.750311:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750327:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828730.750328:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.750332:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828730.750334:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828730.750337:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828730.750340:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273472:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828730.750344:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828730.750346:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828730.750348:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.750350:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828730.750354:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828730.750356:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828730.750360:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828730.750364:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828730.750368:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828730.750371:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273472, portal 10
00000100:00000001:3.0:1787828730.750373:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828730.750375:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:3.0:1787828730.750384:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750391:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828730.750394:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828730.750397:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828730.750400:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327273472, offset 0
00000400:00000200:3.0:1787828730.750404:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828730.750409:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.750414:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828730.750434:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828730.750437:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828730.750440:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828730.750443:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828730.750447:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39908:39908:256:4294967295] 192.168.203.116@tcp LPNI seq info [39908:39908:8:4294967295]
00000400:00000200:3.0:1787828730.750452:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.750460:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828730.750466:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.750470:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828730.750475:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828730.750479:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828730.750483:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828730.750487:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828730.750492:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.750494:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828730.750498:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828730.750503:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828730.750506:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828730.750508:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828730.750511:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828730.750516:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750525:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.750528:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750536:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750538:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828730.750558:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828730.750563:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828730.750567:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828730.750569:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297550859
00000100:00000001:3.0:1787828730.750573:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297551533
00000100:00000001:3.0:1787828730.750576:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297550859
00000100:00000001:3.0:1787828730.750578:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297551533
00000100:00000001:3.0:1787828730.750580:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828730.750583:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750585:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828730.750590:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730]  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828730.750599:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828730.750600:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828730.750603:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828730.750605:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828730.750623:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828730.750625:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828730.750629:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828730.750640:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750648:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828730.750649:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750651:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.750653:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828730.750657:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750665:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828730.750670:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750698:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828730.750701:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750703:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.750704:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.750706:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828730.750709:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828730.750712:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828730.750717:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828730.750719:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828730.750749:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828730.750758:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273472:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828730.750763:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828730.750767:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750775:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828730.750776:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.750779:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828730.750781:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828730.750783:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828730.750787:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273600:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828730.750790:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828730.750793:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828730.750795:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.750797:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828730.750801:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828730.750803:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828730.750807:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828730.750810:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828730.750814:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828730.750818:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273600, portal 4
00000100:00000001:3.0:1787828730.750820:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828730.750822:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:3.0:1787828730.750829:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750836:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828730.750839:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828730.750842:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828730.750844:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327273600, offset 0
00000400:00000200:3.0:1787828730.750849:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828730.750853:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.750857:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828730.750861:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828730.750864:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828730.750867:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828730.750869:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828730.750873:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39909:39909:256:4294967295] 192.168.203.116@tcp LPNI seq info [39909:39909:8:4294967295]
00000400:00000200:3.0:1787828730.750878:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.750885:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828730.750890:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.750894:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828730.750898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828730.750901:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828730.750906:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828730.750909:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828730.750914:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.750916:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828730.750919:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828730.750923:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828730.750926:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828730.750928:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828730.750931:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828730.750935:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750944:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.750947:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.750955:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750957:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828730.750961:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828730.750964:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828730.750968:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828730.750970:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297550867
00000100:00000001:3.0:1787828730.750972:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297551533
00000100:00000001:3.0:1787828730.750975:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297550867
00000100:00000001:3.0:1787828730.750977:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297551533
00000100:00000001:3.0:1787828730.750979:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828730.750982:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.750984:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828730.750988:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730]  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828730.750998:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828730.750999:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828730.751002:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828730.751005:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828730.751006:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828730.751009:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828730.751012:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828730.751022:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751029:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828730.751031:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751033:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.751035:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828730.751039:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751047:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828730.751051:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751060:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828730.751063:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751064:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.751066:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751068:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828730.751070:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828730.751073:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828730.751077:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828730.751079:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828730.751084:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828730.751093:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273600:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828730.751098:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828730.751102:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751110:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828730.751112:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.751114:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828730.751116:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828730.751118:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828730.751122:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273728:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828730.751125:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828730.751127:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828730.751129:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828730.751131:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828730.751135:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828730.751137:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828730.751140:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828730.751144:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828730.751148:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828730.751151:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273728, portal 4
00000100:00000001:3.0:1787828730.751153:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828730.751155:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:3.0:1787828730.751162:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751169:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828730.751172:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828730.751174:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828730.751177:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327273728, offset 0
00000400:00000200:3.0:1787828730.751182:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828730.751186:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.751190:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828730.751194:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828730.751196:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828730.751199:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828730.751202:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828730.751205:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39910:39910:256:4294967295] 192.168.203.116@tcp LPNI seq info [39910:39910:8:4294967295]
00000400:00000200:3.0:1787828730.751210:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828730.751217:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828730.751222:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.751226:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828730.751230:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828730.751242:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828730.751251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828730.751254:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828730.751258:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828730.751261:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828730.751264:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828730.751283:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828730.751287:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828730.751289:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828730.751293:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828730.751297:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751305:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.751309:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751316:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.751318:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828730.751322:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828730.751325:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828730.751329:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828730.751331:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297550872
00000100:00000001:3.0:1787828730.751334:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297551533
00000100:00000001:3.0:1787828730.751337:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297550872
00000100:00000001:3.0:1787828730.751339:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297551533
00000100:00000001:3.0:1787828730.751342:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828730.751345:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.751347:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828730.751352:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730]  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828730.751361:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828730.751362:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828730.751364:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828730.751367:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828730.751369:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828730.751371:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828730.751375:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828730.751385:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751400:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828730.751401:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751403:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.751405:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828730.751409:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751417:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828730.751437:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751445:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828730.751448:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751450:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828730.751451:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751454:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828730.751456:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828730.751459:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828730.751463:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828730.751466:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828730.751470:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828730.751480:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273728:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828730.751485:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751487:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.751491:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327273472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751500:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828730.751502:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828730.751505:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828730.751508:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828730.751510:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828730.751512:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828730.751514:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828730.751516:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:3.0:1787828730.751519:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828730.751521:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828730.751523:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:3.0:1787828730.751527:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828730.751528:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751530:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.751535:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327273600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751559:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828730.751562:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828730.751563:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828730.751566:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828730.751567:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828730.751569:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828730.751571:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828730.751573:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828730.751575:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828730.751577:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828730.751578:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:3.0:1787828730.751581:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828730.751582:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751584:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828730.751588:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327273728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828730.751595:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828730.751597:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828730.751599:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828730.751602:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828730.751603:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828730.751605:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828730.751623:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828730.751625:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828730.751627:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828730.751629:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828730.751630:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828730.751633:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828730.751634:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751636:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828730.751641:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828730.751642:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828730.751644:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828730.751647:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828731.452763:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297551533, at 4297551552
00000100:00000040:2.0:1787828731.452779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297551533, at 4297551552
00000100:00000010:3.0:1787828731.452781:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f780.
00000100:00000010:2.0:1787828731.452793:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f180.
00000020:00000001:2.0:1787828731.580866:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828731.580873:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828731.580878:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828733.628848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828733.628851:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828733.628854:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828734.652919:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828734.652924:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828734.652928:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828735.868759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828735.868767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828735.868769:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828735.868772:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828735.868776:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828735.868782:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2883
00000400:00000200:2.0:1787828735.868789:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828735.868794:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828735.868797:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828735.868799:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868801:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.868803:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868804:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828735.868808:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828735.868811:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828735.868814:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828735.868817:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828735.868820:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828735.868823:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828735.868826:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868829:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.868831:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868833:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828735.868837:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828735.868841:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868844:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828735.868848:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828735.868851:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828735.868855:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.868857:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828735.868860:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828735.868869:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787828735.868873:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828735.868877:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828735.868879:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.868882:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828735.868885:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828735.868889:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828735.868896:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828735.868902:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828735.868910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828735.868918:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828735.868921:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828735.868925:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828735.868948:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828735.868951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828735.868953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828735.868954:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828735.868956:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828735.868958:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828735.868959:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2883
00000400:00000200:2.0:1787828735.868962:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828735.868963:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828735.868965:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828735.868967:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868969:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.868970:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868971:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828735.868973:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828735.868973:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828735.868975:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828735.868976:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828735.868977:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828735.868979:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828735.868980:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868982:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.868983:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868984:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828735.868986:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828735.868988:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828735.868989:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828735.868991:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828735.868996:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828735.868997:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.868999:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828735.869002:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828735.869004:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869006:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828735.869011:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828735.869014:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828735.869016:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828735.869017:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869019:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828735.869020:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828735.869021:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828735.869023:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828735.869027:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828735.869031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828735.869037:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828735.869038:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828735.869041:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.869045:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828735.869047:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828735.869050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828735.869056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828735.869057:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828735.869058:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828735.869059:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828735.869061:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787828735.869068:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787828735.869069:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2883
00000400:00000200:2.0:1787828735.869093:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828735.869100:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828735.869103:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828735.869103:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828735.869114:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869121:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828735.869146:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787828735.869147:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828735.869153:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869155:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:3.0:1787828735.869161:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1787828735.869163:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828735.869167:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828735.869169:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:3.0:1787828735.869169:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1787828735.869175:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828735.869181:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
02000000:00000001:3.0:1787828735.869181:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828735.869197:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828735.869206:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00100000:3.0:1787828735.869206:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327273856:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828735.869216:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.869222:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869225:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828735.869226:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787828735.869228:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828735.869245:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869264:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.869265:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828735.869267:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828735.869271:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828735.869272:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828735.869283:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.869287:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787828735.869288:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828735.869295:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828735.869298:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869299:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828735.869303:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
02000000:00000010:3.0:1787828735.869305:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
00000020:00000040:2.0:1787828735.869307:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828735.869316:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828735.869317:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828735.869319:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869320:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828735.869321:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828735.869322:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828735.869326:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828735.869329:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828735.869336:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828735.869344:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828735.869349:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828735.869365:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787828735.869367:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828735.869376:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828735.869384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828735.869387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:3.0:1787828735.869388:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000100:00000001:2.0:1787828735.869389:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828735.869393:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828735.869397:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828735.869401:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2883
00000400:00000200:2.0:1787828735.869404:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828735.869405:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000400:00000200:2.0:1787828735.869409:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828735.869412:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828735.869415:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869417:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.869419:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:3.0:1787828735.869419:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273856, portal 25
00000100:00000001:2.0:1787828735.869427:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828735.869430:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:2.0:1787828735.869431:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828735.869438:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:3.0:1787828735.869438:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402436160 : -116696307115456 : ffff95dd86115a40)
00000020:00000001:2.0:1787828735.869447:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828735.869450:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828735.869452:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828735.869453:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828735.869456:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869457:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828735.869459:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869461:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828735.869468:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000040:3.0:1787828735.869469:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828735.869474:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869500:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828735.869508:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828735.869515:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1787828735.869516:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828735.869524:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869526:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828735.869530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1787828735.869533:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000008:00000001:2.0:1787828735.869535:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869542:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828735.869543:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828735.869561:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327273856, offset 0
00000100:00000010:2.0:1787828735.869563:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828735.869569:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828735.869572:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828735.869573:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828735.869575:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828735.869576:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828735.869585:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828735.869591:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000400:00000200:3.0:1787828735.869593:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828735.869601:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828735.869613:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:2.0:1787828735.869632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828735.869642:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828735.869647:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828735.869650:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828735.869656:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1787828735.869657:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828735.869671:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828735.869675:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2893)
00000400:00000200:3.0:1787828735.869736:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828735.869743:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828735.869748:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828735.869753:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39911:39911:256:4294967295] 192.168.203.116@tcp LPNI seq info [39911:39911:8:4294967295]
00000400:00000200:3.0:1787828735.869760:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.869774:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828735.869783:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.869791:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828735.869800:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828735.869815:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828735.869839:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828735.869862:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828735.869870:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.869874:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828735.869885:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828735.869892:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828735.869901:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828735.869905:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828735.869912:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828735.869919:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.869935:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.869940:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.869950:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.869953:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828735.869962:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828735.869969:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828735.869973:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297556652
00000100:00000010:3.0:1787828735.869980:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f780.
00000100:00000001:3.0:1787828735.869989:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828735.870008:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297556652
00000100:00000010:3.0:1787828735.870013:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f1e0.
00000100:00000001:3.0:1787828735.870019:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828735.870025:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828735.870030:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.870035:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828735.870042:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735]  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828735.870056:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828735.870059:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828735.870062:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828735.870066:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828735.870069:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828735.870073:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828735.870078:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828735.870091:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870108:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828735.870112:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870115:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.870118:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828735.870124:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870161:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828735.870168:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870179:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828735.870183:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870185:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.870187:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870190:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828735.870193:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828735.870198:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828735.870203:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828735.870207:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828735.870213:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828735.870225:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327273856:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828735.870250:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828735.870257:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870266:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828735.870268:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.870272:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828735.870276:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828735.870290:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828735.870294:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273984:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828735.870299:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828735.870301:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828735.870304:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.870306:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828735.870311:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828735.870314:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828735.870318:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828735.870322:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828735.870327:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828735.870331:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327273984, portal 10
00000100:00000001:3.0:1787828735.870334:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828735.870336:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:3.0:1787828735.870345:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870354:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828735.870357:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828735.870360:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828735.870364:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327273984, offset 0
00000400:00000200:3.0:1787828735.870369:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828735.870374:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.870379:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828735.870383:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828735.870386:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828735.870389:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828735.870392:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828735.870396:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39912:39912:256:4294967295] 192.168.203.116@tcp LPNI seq info [39912:39912:8:4294967295]
00000400:00000200:3.0:1787828735.870402:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.870410:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828735.870417:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.870422:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828735.870427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828735.870431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828735.870437:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828735.870440:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828735.870445:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.870448:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828735.870452:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828735.870457:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828735.870461:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828735.870463:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828735.870466:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828735.870471:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870481:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.870486:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870503:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.870505:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828735.870509:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828735.870514:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828735.870519:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828735.870522:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297555979
00000100:00000001:3.0:1787828735.870526:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297556652
00000100:00000001:3.0:1787828735.870529:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297555979
00000100:00000001:3.0:1787828735.870532:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297556652
00000100:00000001:3.0:1787828735.870535:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828735.870537:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.870540:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828735.870544:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735]  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828735.870575:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828735.870577:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828735.870579:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828735.870582:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828735.870585:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828735.870587:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828735.870591:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828735.870603:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870612:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828735.870614:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870616:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.870618:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828735.870623:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870632:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828735.870637:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870646:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828735.870649:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870652:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.870653:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870680:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828735.870683:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828735.870687:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828735.870691:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828735.870695:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828735.870701:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828735.870711:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327273984:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828735.870716:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870719:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.870723:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115a40 x1874671327273856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870733:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828735.870737:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828735.870741:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828735.870745:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828735.870749:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828735.870751:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828735.870754:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828735.870757:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:3.0:1787828735.870762:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828735.870765:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828735.870768:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115a40.
00000100:00000001:3.0:1787828735.870772:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828735.870774:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870776:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.870781:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327273984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870789:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828735.870791:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828735.870793:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828735.870795:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828735.870797:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828735.870799:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828735.870801:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828735.870803:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828735.870806:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828735.870808:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828735.870811:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828735.870814:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828735.870816:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870819:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.870830:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828735.870833:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.870836:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828735.870839:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828735.870841:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828735.870847:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870856:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828735.870858:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.870861:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828735.870863:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828735.870866:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828735.870870:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274112:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828735.870875:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828735.870877:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828735.870880:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.870882:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828735.870885:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828735.870888:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828735.870892:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828735.870895:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828735.870900:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828735.870903:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274112, portal 4
00000100:00000001:3.0:1787828735.870905:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828735.870907:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:3.0:1787828735.870915:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.870924:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828735.870928:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828735.870930:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828735.870934:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327274112, offset 0
00000400:00000200:3.0:1787828735.870939:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828735.870944:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.870949:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828735.870953:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828735.870956:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828735.870960:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828735.870963:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828735.870967:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39913:39913:256:4294967295] 192.168.203.116@tcp LPNI seq info [39913:39913:8:4294967295]
00000400:00000200:3.0:1787828735.870972:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.870980:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828735.870987:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.870991:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828735.870996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828735.871000:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828735.871006:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828735.871010:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828735.871015:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.871018:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828735.871022:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828735.871026:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828735.871030:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828735.871032:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828735.871036:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828735.871041:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871051:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.871056:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871065:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.871067:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828735.871072:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828735.871077:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828735.871082:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828735.871084:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297555987
00000100:00000001:3.0:1787828735.871087:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297556652
00000100:00000001:3.0:1787828735.871091:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297555987
00000100:00000001:3.0:1787828735.871093:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297556652
00000100:00000001:3.0:1787828735.871095:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828735.871099:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.871101:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828735.871106:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735]  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828735.871116:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828735.871118:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828735.871120:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828735.871123:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828735.871126:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828735.871128:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828735.871133:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828735.871144:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871154:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828735.871156:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871159:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.871161:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828735.871166:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871174:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828735.871178:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871187:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828735.871191:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871193:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.871195:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871197:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828735.871199:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828735.871202:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828735.871206:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828735.871210:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828735.871215:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828735.871227:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274112:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828735.871249:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828735.871255:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871264:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828735.871266:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.871269:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828735.871272:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828735.871275:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828735.871279:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274240:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828735.871283:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828735.871285:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828735.871288:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828735.871290:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828735.871294:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828735.871297:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828735.871301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828735.871305:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828735.871308:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828735.871313:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274240, portal 4
00000100:00000001:3.0:1787828735.871315:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828735.871316:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:3.0:1787828735.871324:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871333:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828735.871336:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828735.871339:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828735.871342:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327274240, offset 0
00000400:00000200:3.0:1787828735.871348:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828735.871353:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.871358:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828735.871361:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828735.871365:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828735.871368:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828735.871372:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828735.871376:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39914:39914:256:4294967295] 192.168.203.116@tcp LPNI seq info [39914:39914:8:4294967295]
00000400:00000200:3.0:1787828735.871381:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828735.871389:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828735.871395:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.871399:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828735.871404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828735.871408:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828735.871413:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828735.871417:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828735.871422:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828735.871425:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828735.871429:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828735.871434:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828735.871438:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828735.871440:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828735.871444:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828735.871449:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871459:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.871463:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871471:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.871474:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828735.871478:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828735.871482:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828735.871486:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828735.871488:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297555992
00000100:00000001:3.0:1787828735.871491:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297556652
00000100:00000001:3.0:1787828735.871493:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297555992
00000100:00000001:3.0:1787828735.871495:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297556652
00000100:00000001:3.0:1787828735.871497:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828735.871498:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.871500:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828735.871504:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735]  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828735.871509:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828735.871511:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828735.871512:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828735.871514:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828735.871516:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828735.871518:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828735.871520:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828735.871527:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871538:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828735.871539:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871541:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.871542:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828735.871545:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871550:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828735.871553:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871558:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828735.871559:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871560:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828735.871561:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871563:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828735.871565:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828735.871567:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828735.871570:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828735.871572:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828735.871574:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828735.871580:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274240:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828735.871583:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871585:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.871587:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327274112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871592:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828735.871593:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828735.871596:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828735.871598:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828735.871599:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828735.871600:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828735.871602:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828735.871603:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:3.0:1787828735.871605:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828735.871607:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828735.871608:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:3.0:1787828735.871610:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828735.871611:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871613:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828735.871615:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327274240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828735.871620:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828735.871621:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828735.871622:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828735.871623:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828735.871624:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828735.871625:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828735.871626:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828735.871627:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828735.871628:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828735.871629:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828735.871630:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:3.0:1787828735.871632:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828735.871633:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871635:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828735.871638:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828735.871639:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828735.871641:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828735.871642:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828736.572818:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297556652, at 4297556672
00000100:00000040:3.0:1787828736.572818:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297556652, at 4297556672
00000100:00000010:2.0:1787828736.572846:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f780.
00000100:00000010:3.0:1787828736.572850:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f1e0.
00000020:00000001:3.0:1787828736.700869:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828736.700873:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828736.700877:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828737.724902:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828737.724907:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828737.724911:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828739.772940:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828739.772945:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828739.772949:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828740.796868:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828740.796871:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828740.796874:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828740.988787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828740.988796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828740.988798:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828740.988801:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828740.988806:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828740.988812:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2888
00000400:00000200:2.0:1787828740.988821:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828740.988844:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828740.988849:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828740.988852:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.988855:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828740.988857:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.988860:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828740.988864:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828740.988866:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828740.988868:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828740.988872:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828740.988874:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828740.988877:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828740.988880:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.988881:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828740.988883:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.988885:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828740.988888:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828740.988892:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828740.988894:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828740.988898:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828740.988901:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828740.988904:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.988906:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828740.988909:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828740.988919:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828740.988923:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828740.988928:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828740.988946:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.988949:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828740.988952:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828740.988956:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828740.988964:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828740.988969:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828740.988978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828740.988989:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828740.988993:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828740.989013:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828740.989046:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828740.989051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828740.989056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828740.989058:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828740.989061:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828740.989065:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828740.989068:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2888
00000400:00000200:2.0:1787828740.989074:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828740.989095:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828740.989097:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828740.989100:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989102:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828740.989104:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989106:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828740.989109:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828740.989121:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828740.989123:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828740.989125:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828740.989128:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828740.989130:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:3.0:1787828740.989130:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787828740.989133:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989135:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828740.989135:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787828740.989137:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828740.989137:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787828740.989139:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828740.989142:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828740.989144:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:3.0:1787828740.989144:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828740.989147:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828740.989149:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828740.989152:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828740.989154:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828740.989155:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828740.989157:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.989158:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787828740.989158:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828740.989160:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:2.0:1787828740.989163:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787828740.989163:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787828740.989166:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828740.989166:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327274368:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828740.989168:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.989171:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1787828740.989172:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828740.989174:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828740.989175:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828740.989176:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828740.989177:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.989178:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.989179:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828740.989180:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787828740.989181:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828740.989182:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828740.989185:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000010:2.0:1787828740.989187:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828740.989187:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.989191:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828740.989193:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:2.0:1787828740.989211:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787828740.989211:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828740.989229:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000001:2.0:1787828740.989230:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828740.989249:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828740.989253:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274368, portal 25
00000100:00000040:2.0:1787828740.989255:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989256:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828740.989259:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787828740.989268:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828740.989268:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828740.989272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828740.989277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828740.989279:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787828740.989280:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787828740.989282:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1787828740.989284:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989286:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:3.0:1787828740.989288:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00080000:2.0:1787828740.989289:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2888
00000100:00000200:3.0:1787828740.989292:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327274368, offset 0
00000400:00000200:2.0:1787828740.989295:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828740.989300:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:3.0:1787828740.989302:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787828740.989304:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828740.989309:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989312:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787828740.989313:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787828740.989316:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989319:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:3.0:1787828740.989321:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000001:2.0:1787828740.989323:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828740.989325:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828740.989328:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787828740.989330:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1787828740.989331:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828740.989334:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:3.0:1787828740.989336:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:2.0:1787828740.989338:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828740.989342:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828740.989342:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787828740.989344:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787828740.989347:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:2.0:1787828740.989348:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989350:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:3.0:1787828740.989353:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39915:39915:256:4294967295] 192.168.203.116@tcp LPNI seq info [39915:39915:8:4294967295]
00000020:00000040:2.0:1787828740.989355:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828740.989359:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:3.0:1787828740.989360:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828740.989363:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828740.989365:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828740.989369:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828740.989372:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828740.989374:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787828740.989375:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828740.989378:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828740.989382:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787828740.989383:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989385:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787828740.989390:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000100:00000010:2.0:1787828740.989391:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828740.989394:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828740.989398:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828740.989400:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787828740.989400:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828740.989403:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828740.989405:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1787828740.989406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
02000000:00000001:2.0:1787828740.989408:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828740.989414:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000800:00000200:3.1:1787828740.989415:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828740.989418:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828740.989420:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00100000:2.0:1787828740.989425:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:3.0:1787828740.989427:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.989432:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000100:00000001:2.0:1787828740.989436:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828740.989439:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828740.989441:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1787828740.989444:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787828740.989448:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828740.989455:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000100:00000001:2.0:1787828740.989456:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828740.989458:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00080000:2.0:1787828740.989460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828740.989464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828740.989464:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787828740.989467:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828740.989469:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1787828740.989470:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828740.989489:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828740.989492:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2888
00000400:00000200:2.0:1787828740.989497:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828740.989499:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:2.0:1787828740.989500:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828740.989503:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828740.989504:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828740.989508:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989511:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828740.989514:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828740.989515:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.989517:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828740.989518:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:2.0:1787828740.989521:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828740.989522:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828740.989524:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787828740.989525:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787828740.989527:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828740.989530:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:3.0:1787828740.989530:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787828740.989533:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:3.0:1787828740.989534:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297561772
00000100:00000040:2.0:1787828740.989538:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989540:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787828740.989541:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f1e0.
00000100:00000040:2.0:1787828740.989560:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989563:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828740.989566:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:3.0:1787828740.989566:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:2.0:1787828740.989569:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828740.989573:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.989573:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297561772
02000000:00000001:2.0:1787828740.989577:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:3.0:1787828740.989578:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f540.
02000000:00000001:2.0:1787828740.989581:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828740.989583:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:2.0:1787828740.989584:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.989587:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828740.989587:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00100000:2.0:1787828740.989590:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828740.989592:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787828740.989593:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.989595:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787828740.989596:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828740.989600:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000100:00100000:3.0:1787828740.989602:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740]  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787828740.989604:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828740.989608:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828740.989610:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828740.989613:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828740.989615:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:3.0:1787828740.989617:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:2.0:1787828740.989618:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787828740.989619:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000010:2.0:1787828740.989623:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000400:00000010:3.0:1787828740.989623:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828740.989627:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000100:00000001:2.0:1787828740.989628:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828740.989630:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828740.989634:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:2.0:1787828740.989635:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787828740.989639:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828740.989647:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828740.989650:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828740.989655:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828740.989656:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989667:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787828740.989669:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.989670:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.989673:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1787828740.989675:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2898)
00000100:00000001:3.0:1787828740.989676:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828740.989682:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989693:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828740.989744:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989751:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828740.989754:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.989755:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.989756:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.989758:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828740.989760:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828740.989762:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828740.989765:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828740.989766:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828740.989769:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828740.989775:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327274368:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828740.989779:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.989780:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.989782:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327274368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989786:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828740.989788:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828740.989790:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828740.989792:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828740.989794:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828740.989795:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828740.989796:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828740.989798:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:3.0:1787828740.989800:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828740.989802:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828740.989803:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:3.0:1787828740.989817:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828740.989818:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.989819:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.989843:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828740.989844:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.989846:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828740.989848:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828740.989849:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828740.989853:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989860:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828740.989861:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.989863:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828740.989865:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828740.989867:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828740.989869:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274496:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828740.989873:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828740.989875:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828740.989876:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.989878:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828740.989881:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828740.989883:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828740.989887:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828740.989889:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828740.989893:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828740.989896:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274496, portal 10
00000100:00000001:3.0:1787828740.989898:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828740.989899:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:3.0:1787828740.989905:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.989911:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828740.989913:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828740.989915:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828740.989918:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327274496, offset 0
00000400:00000200:3.0:1787828740.989921:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828740.989925:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828740.989929:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828740.989931:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828740.989934:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828740.989937:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828740.989939:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828740.989942:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39916:39916:256:4294967295] 192.168.203.116@tcp LPNI seq info [39916:39916:8:4294967295]
00000400:00000200:3.0:1787828740.989946:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828740.989952:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828740.989957:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.989960:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828740.989964:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828740.989995:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828740.989999:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828740.990002:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787828740.990006:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.990008:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828740.990012:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828740.990016:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828740.990019:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828740.990021:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828740.990023:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828740.990027:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990034:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.990037:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990044:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990045:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828740.990049:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828740.990052:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828740.990055:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828740.990058:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297561099
00000100:00000001:3.0:1787828740.990061:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297561772
00000100:00000001:3.0:1787828740.990063:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297561099
00000100:00000001:3.0:1787828740.990081:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297561772
00000100:00000001:3.0:1787828740.990083:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828740.990085:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990086:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828740.990088:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740]  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828740.990093:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828740.990094:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828740.990095:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828740.990097:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828740.990098:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828740.990099:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828740.990101:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828740.990107:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990111:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828740.990112:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990113:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.990114:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828740.990116:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990120:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828740.990122:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990126:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828740.990128:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990129:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.990146:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990148:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828740.990150:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828740.990152:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828740.990156:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828740.990158:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828740.990162:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828740.990169:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274496:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828740.990172:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828740.990175:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990181:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828740.990182:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.990184:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828740.990185:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828740.990187:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828740.990190:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274624:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828740.990192:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828740.990213:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828740.990232:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.990233:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828740.990236:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828740.990237:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828740.990240:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828740.990241:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828740.990244:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828740.990246:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274624, portal 4
00000100:00000001:3.0:1787828740.990247:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828740.990248:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:3.0:1787828740.990252:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990261:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828740.990262:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828740.990264:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828740.990265:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327274624, offset 0
00000400:00000200:3.0:1787828740.990268:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828740.990270:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828740.990273:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828740.990274:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828740.990276:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828740.990277:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828740.990279:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828740.990281:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39917:39917:256:4294967295] 192.168.203.116@tcp LPNI seq info [39917:39917:8:4294967295]
00000400:00000200:3.0:1787828740.990284:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828740.990288:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828740.990291:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.990293:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828740.990295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828740.990297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828740.990300:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828740.990301:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787828740.990304:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.990305:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828740.990307:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828740.990310:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828740.990311:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828740.990313:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828740.990314:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828740.990316:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990322:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.990324:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990332:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990334:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828740.990335:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828740.990337:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828740.990339:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828740.990340:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297561106
00000100:00000001:3.0:1787828740.990342:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297561772
00000100:00000001:3.0:1787828740.990344:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297561106
00000100:00000001:3.0:1787828740.990345:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297561772
00000100:00000001:3.0:1787828740.990346:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828740.990347:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990349:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828740.990351:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740]  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828740.990356:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828740.990357:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828740.990358:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828740.990360:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828740.990361:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828740.990362:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828740.990364:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828740.990369:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990372:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828740.990373:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990374:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.990375:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828740.990377:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990382:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828740.990385:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990389:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828740.990390:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990391:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.990392:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990393:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828740.990394:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828740.990396:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828740.990397:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828740.990399:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828740.990401:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828740.990405:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274624:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828740.990408:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828740.990410:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990413:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828740.990415:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.990416:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828740.990417:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828740.990419:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828740.990420:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274752:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828740.990422:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828740.990423:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828740.990425:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828740.990426:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828740.990428:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828740.990429:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828740.990431:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828740.990433:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828740.990435:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828740.990436:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274752, portal 4
00000100:00000001:3.0:1787828740.990438:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828740.990439:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:3.0:1787828740.990442:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990456:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828740.990457:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828740.990458:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828740.990460:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327274752, offset 0
00000400:00000200:3.0:1787828740.990462:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828740.990464:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828740.990466:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828740.990468:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828740.990470:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828740.990471:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828740.990473:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828740.990475:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39918:39918:256:4294967295] 192.168.203.116@tcp LPNI seq info [39918:39918:8:4294967295]
00000400:00000200:3.0:1787828740.990477:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828740.990481:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828740.990484:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.990486:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828740.990488:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828740.990490:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828740.990493:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828740.990495:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787828740.990497:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828740.990499:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828740.990501:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828740.990503:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828740.990505:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828740.990506:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828740.990507:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828740.990510:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990515:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.990528:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990532:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990533:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828740.990534:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828740.990536:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828740.990538:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828740.990539:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297561111
00000100:00000001:3.0:1787828740.990541:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297561772
00000100:00000001:3.0:1787828740.990542:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297561111
00000100:00000001:3.0:1787828740.990544:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297561772
00000100:00000001:3.0:1787828740.990545:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828740.990546:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990547:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828740.990549:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740]  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828740.990554:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828740.990554:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828740.990555:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828740.990557:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828740.990558:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828740.990559:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828740.990561:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828740.990567:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990570:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828740.990571:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990572:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.990573:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828740.990575:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990579:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828740.990581:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990585:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828740.990586:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990587:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828740.990588:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990589:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828740.990590:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828740.990592:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828740.990593:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828740.990595:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828740.990597:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828740.990601:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327274752:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828740.990603:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990605:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.990607:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327274496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990611:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828740.990612:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828740.990614:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828740.990615:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828740.990616:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828740.990617:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828740.990618:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828740.990619:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828740.990621:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828740.990621:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828740.990623:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:3.0:1787828740.990624:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828740.990625:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990626:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.990628:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327274624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990632:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828740.990633:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828740.990634:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828740.990635:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828740.990636:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828740.990637:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828740.990638:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828740.990639:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:3.0:1787828740.990640:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828740.990640:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828740.990641:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:3.0:1787828740.990643:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828740.990644:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990644:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828740.990646:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327274752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828740.990650:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828740.990651:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828740.990652:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828740.990653:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828740.990654:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828740.990655:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828740.990656:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828740.990657:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828740.990658:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828740.990659:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828740.990659:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:3.0:1787828740.990661:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828740.990661:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990662:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828740.990664:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828740.990665:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828740.990666:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828740.990668:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828741.692756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297561772, at 4297561792
00000100:00000040:3.0:1787828741.692756:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297561772, at 4297561792
00000100:00000010:3.0:1787828741.692769:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f540.
00000100:00000010:2.0:1787828741.692773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f1e0.
00000020:00000001:3.0:1787828742.844895:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828742.844899:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828742.844902:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828743.869000:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828743.869018:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828743.869024:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828745.916957:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828745.916963:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828745.916967:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828746.108852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828746.108872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828746.108877:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828746.108884:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828746.108909:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828746.108923:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2893
00000400:00000200:2.0:1787828746.108943:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828746.108957:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828746.108965:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828746.108972:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.108979:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828746.108999:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109004:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828746.109012:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828746.109015:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828746.109020:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828746.109027:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828746.109031:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828746.109035:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828746.109041:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109045:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828746.109064:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109068:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828746.109075:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828746.109083:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109089:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828746.109095:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828746.109102:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828746.109109:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.109114:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828746.109120:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828746.109138:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828746.109146:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828746.109155:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828746.109183:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.109192:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828746.109198:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828746.109206:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828746.109222:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828746.109237:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828746.109256:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828746.109279:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828746.109287:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828746.109298:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828746.109361:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828746.109367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828746.109372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828746.109375:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828746.109378:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828746.109382:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828746.109387:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2893
00000400:00000200:2.0:1787828746.109393:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828746.109399:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828746.109403:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828746.109426:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109433:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828746.109438:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109444:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828746.109449:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787828746.109453:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787828746.109454:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828746.109462:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787828746.109469:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1787828746.109471:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:3.0:1787828746.109480:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787828746.109482:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828746.109492:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828746.109504:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828746.109506:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109517:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828746.109541:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109550:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828746.109555:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787828746.109558:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:3.0:1787828746.109561:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828746.109569:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:3.0:1787828746.109574:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787828746.109577:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.109582:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828746.109584:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828746.109593:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828746.109598:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787828746.109603:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327274880:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787828746.109606:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.109618:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828746.109618:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828746.109627:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1787828746.109630:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787828746.109631:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.109639:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:2.0:1787828746.109642:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.109649:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828746.109649:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828746.109655:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828746.109659:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000400:00000200:3.0:1787828746.109666:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787828746.109669:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828746.109676:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828746.109679:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.109683:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828746.109687:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828746.109690:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828746.109698:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000400:00000010:3.0:1787828746.109765:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828746.109779:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000001:2.0:1787828746.109782:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828746.109794:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327274880, portal 25
00000100:00100000:2.0:1787828746.109803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828746.109804:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828746.109810:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000100:00000040:3.0:1787828746.109847:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828746.109851:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828746.109859:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.109864:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828746.109874:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787828746.109876:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787828746.109878:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828746.109886:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327274880, offset 0
00000400:00000200:3.0:1787828746.109904:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828746.109909:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828746.109916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787828746.109922:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828746.109927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828746.109933:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787828746.109934:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:2.0:1787828746.109937:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787828746.109944:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787828746.109947:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787828746.109953:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:2.0:1787828746.109957:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2893
00000400:00000200:3.0:1787828746.109964:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828746.109969:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828746.109974:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39919:39919:256:4294967295] 192.168.203.116@tcp LPNI seq info [39919:39919:8:4294967295]
00000400:00000200:3.0:1787828746.109979:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.109991:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787828746.110002:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:3.0:1787828746.110003:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828746.110013:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000400:00000200:2.0:1787828746.110016:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828746.110023:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787828746.110027:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828746.110054:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:3.0:1787828746.110056:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787828746.110066:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1787828746.110076:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828746.110083:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:3.1:1787828746.110086:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787828746.110097:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000100:3.0:1787828746.110104:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787828746.110111:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:3.0:1787828746.110113:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000020:00000040:2.0:1787828746.110121:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828746.110129:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787828746.110131:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1787828746.110137:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:3.0:1787828746.110144:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1787828746.110151:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:3.0:1787828746.110156:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000020:00000001:2.0:1787828746.110193:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:3.0:1787828746.110202:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000040:2.0:1787828746.110211:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828746.110218:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787828746.110221:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828746.110236:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:3.0:1787828746.110239:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828746.110243:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828746.110257:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828746.110273:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828746.110285:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.110287:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787828746.110292:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787828746.110300:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787828746.110304:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828746.110318:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.110330:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.110331:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828746.110336:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000008:00100000:2.0:1787828746.110342:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828746.110365:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.110367:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.110369:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828746.110377:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:2.0:1787828746.110380:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000100:00000001:3.0:1787828746.110382:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297566893
00000020:00000040:2.0:1787828746.110389:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000010:3.0:1787828746.110395:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f540.
02000000:00000001:2.0:1787828746.110398:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828746.110403:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.110408:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787828746.110425:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828746.110434:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787828746.110439:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297566893
02000000:00000001:2.0:1787828746.110443:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828746.110447:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f120.
00000100:00000001:3.0:1787828746.110456:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000010:2.0:1787828746.110460:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:3.0:1787828746.110461:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828746.110472:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828746.110478:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.110479:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828746.110490:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828746/real 1787828746]  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828746.110501:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:3.0:1787828746.110511:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828746.110516:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828746.110523:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828746.110546:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828746.110551:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:2.0:1787828746.110552:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828746.110554:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787828746.110561:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828746.110564:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828746.110577:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828746.110582:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.110618:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828746.110621:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828746.110627:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.110627:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.110631:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787828746.110637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1787828746.110639:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828746.110655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828746.110658:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828746.110661:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1787828746.110668:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828746.110669:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828746.110681:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787828746.110687:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787828746.110711:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2893
00000100:00000001:3.0:1787828746.110713:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.110746:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.110748:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.110752:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828746.110755:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828746.110760:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828746.110770:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828746.110777:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:2.0:1787828746.110786:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787828746.110790:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787828746.110803:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00100000:3.0:1787828746.110833:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327274880:192.168.203.116@tcp:250:kworker.0
00000100:00080000:2.0:1787828746.110834:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828746.110848:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828746.110849:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.110857:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787828746.110858:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828746.110865:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327274880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828746.110870:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828746.110880:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828746.110882:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828746.110906:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000001:2.0:1787828746.110908:0:37:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000010:3.0:1787828746.110912:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
00000020:00000040:2.0:1787828746.110916:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
02000000:00000001:3.0:1787828746.110921:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787828746.110925:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000001:3.0:1787828746.110928:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828746.110935:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000040:2.0:1787828746.110937:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828746.110939:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828746.110948:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
00000020:00000001:2.0:1787828746.110950:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
02000000:00000001:3.0:1787828746.110956:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000001:2.0:1787828746.110959:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
02000000:00000001:3.0:1787828746.110962:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828746.110967:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000040:2.0:1787828746.110992:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828746.110992:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828746.110996:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828746.111002:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828746.111003:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828746.111014:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828746.111018:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828746.111020:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828746.111023:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787828746.111029:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:3.0:1787828746.111030:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828746.111033:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828746.111055:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787828746.111061:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:3.0:1787828746.111067:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828746.111075:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828746.111086:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828746.111094:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828746.111097:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828746.111098:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828746.111101:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828746.111107:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828746.111113:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.111114:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828746.111127:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275008:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787828746.111129:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828746.111133:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:2.0:1787828746.111140:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787828746.111140:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828746.111144:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787828746.111153:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.111154:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828746.111185:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:2.0:1787828746.111191:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.111195:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828746.111207:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000400:00000200:3.0:1787828746.111211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787828746.111219:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000010:3.0:1787828746.111223:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
02000000:00000001:2.0:1787828746.111232:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:3.0:1787828746.111236:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
02000000:00000001:2.0:1787828746.111240:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828746.111248:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275008, portal 10
00000100:00000001:2.0:1787828746.111250:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787828746.111257:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1787828746.111260:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787828746.111262:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
02000000:00000001:2.0:1787828746.111270:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828746.111280:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787828746.111289:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:3.0:1787828746.111293:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828746.111301:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828746.111304:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828746.111307:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828746.111316:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327275008, offset 0
00000400:00000200:3.0:1787828746.111325:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00100000:2.0:1787828746.111327:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828746.111337:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.111366:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828746.111376:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787828746.111383:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787828746.111384:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828746.111393:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787828746.111394:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828746.111397:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828746.111408:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39920:39920:256:4294967295] 192.168.203.116@tcp LPNI seq info [39920:39920:8:4294967295]
00000100:00000040:2.0:1787828746.111411:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787828746.111433:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.111458:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828746.111480:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828746.111485:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787828746.111487:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000100:00000040:2.0:1787828746.111502:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2903)
00000800:00000200:3.0:1787828746.111503:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828746.111515:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828746.111542:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828746.111546:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828746.111554:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828746.111558:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828746.111564:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828746.111570:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828746.111574:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828746.111577:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828746.111581:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828746.111587:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111600:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828746.111606:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111617:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.111619:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828746.111625:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828746.111631:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828746.111635:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828746.111637:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297566220
00000100:00000001:3.0:1787828746.111640:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297566893
00000100:00000001:3.0:1787828746.111643:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297566220
00000100:00000001:3.0:1787828746.111645:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297566893
00000100:00000001:3.0:1787828746.111648:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828746.111650:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.111652:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828746.111656:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828746/real 1787828746]  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828746.111664:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828746.111665:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828746.111667:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828746.111669:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828746.111671:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828746.111674:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828746.111677:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828746.111704:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111713:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828746.111742:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.111746:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.111748:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828746.111753:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111761:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828746.111766:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111774:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828746.111777:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.111779:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.111781:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.111783:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828746.111786:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828746.111789:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828746.111793:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828746.111795:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828746.111801:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828746.111811:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275008:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828746.111833:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828746.111836:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111849:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828746.111851:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.111854:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828746.111856:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828746.111858:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828746.111861:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275136:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828746.111863:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828746.111865:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828746.111867:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.111869:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828746.111873:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828746.111875:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828746.111895:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828746.111899:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828746.111904:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828746.111908:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275136, portal 4
00000100:00000001:3.0:1787828746.111911:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828746.111913:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:3.0:1787828746.111921:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.111930:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828746.111933:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828746.111936:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828746.111939:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327275136, offset 0
00000400:00000200:3.0:1787828746.111944:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828746.111949:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.111954:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828746.111958:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828746.111962:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828746.111965:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828746.111968:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828746.111973:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39921:39921:256:4294967295] 192.168.203.116@tcp LPNI seq info [39921:39921:8:4294967295]
00000400:00000200:3.0:1787828746.111979:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.111988:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828746.112010:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828746.112013:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828746.112017:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828746.112020:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828746.112024:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828746.112027:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828746.112031:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828746.112033:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828746.112037:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828746.112041:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828746.112044:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828746.112045:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828746.112048:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828746.112051:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112078:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828746.112082:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112091:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.112093:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828746.112097:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828746.112101:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828746.112105:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828746.112109:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297566228
00000100:00000001:3.0:1787828746.112112:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297566893
00000100:00000001:3.0:1787828746.112115:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297566228
00000100:00000001:3.0:1787828746.112118:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297566893
00000100:00000001:3.0:1787828746.112121:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828746.112124:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.112126:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828746.112131:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828746/real 1787828746]  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828746.112141:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828746.112143:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828746.112145:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828746.112147:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828746.112149:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828746.112152:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828746.112156:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828746.112201:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112214:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828746.112218:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112222:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.112225:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828746.112233:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112247:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828746.112254:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112264:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828746.112267:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112269:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.112271:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112273:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828746.112275:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828746.112278:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828746.112283:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828746.112286:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828746.112291:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828746.112301:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275136:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828746.112306:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112308:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828746.112312:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327275008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112319:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828746.112322:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828746.112324:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828746.112328:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828746.112330:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828746.112332:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828746.112334:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828746.112337:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828746.112340:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828746.112358:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828746.112360:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:3.0:1787828746.112363:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828746.112364:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112366:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828746.112369:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327275136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112375:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828746.112376:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828746.112378:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828746.112380:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828746.112381:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828746.112382:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828746.112383:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828746.112386:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:3.0:1787828746.112388:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828746.112389:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828746.112390:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:3.0:1787828746.112393:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828746.112394:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112396:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112399:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828746.112400:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.112402:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828746.112404:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828746.112405:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828746.112427:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112436:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828746.112438:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.112441:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828746.112443:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828746.112446:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828746.112449:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275264:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828746.112454:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828746.112457:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828746.112459:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828746.112461:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828746.112466:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828746.112468:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828746.112473:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828746.112475:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828746.112480:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828746.112484:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275264, portal 4
00000100:00000001:3.0:1787828746.112487:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828746.112489:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:3.0:1787828746.112496:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112514:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828746.112519:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828746.112524:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828746.112529:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327275264, offset 0
00000400:00000200:3.0:1787828746.112553:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828746.112560:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.112565:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828746.112569:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828746.112573:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828746.112577:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828746.112581:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828746.112586:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39922:39922:256:4294967295] 192.168.203.116@tcp LPNI seq info [39922:39922:8:4294967295]
00000400:00000200:3.0:1787828746.112594:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828746.112604:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828746.112611:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828746.112617:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828746.112623:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828746.112626:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828746.112630:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828746.112633:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828746.112637:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828746.112639:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828746.112643:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828746.112646:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828746.112650:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828746.112652:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828746.112654:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828746.112658:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112666:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828746.112669:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828771 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112683:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.112684:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828746.112705:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828746.112709:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828746.112714:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828746.112744:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297566234
00000100:00000001:3.0:1787828746.112749:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297566893
00000100:00000001:3.0:1787828746.112752:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297566234
00000100:00000001:3.0:1787828746.112755:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297566893
00000100:00000001:3.0:1787828746.112758:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828746.112761:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.112763:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828746.112769:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828746/real 1787828746]  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828746.112778:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828746.112781:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828746.112783:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828746.112786:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828746.112789:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828746.112792:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828746.112796:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828746.112806:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112832:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828746.112833:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112835:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.112836:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828746.112839:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112847:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828746.112851:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112858:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828746.112860:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112861:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828746.112863:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112864:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828746.112866:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828746.112868:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828746.112871:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828746.112873:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828746.112876:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828746.112901:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275264:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828746.112906:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112908:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828746.112912:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327275264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828771 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828746.112919:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828746.112922:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828746.112924:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828746.112928:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828746.112930:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828746.112932:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828746.112934:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828746.112937:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:3.0:1787828746.112940:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828746.112942:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828746.112944:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:3.0:1787828746.112948:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828746.112949:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112952:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828746.112956:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828746.112958:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828746.112961:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828746.112963:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828746.812777:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297566893, at 4297566912
00000100:00000040:3.0:1787828746.812777:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297566893, at 4297566912
00000100:00000010:2.0:1787828746.812805:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f540.
00000100:00000010:3.0:1787828746.812810:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f120.
00000020:00000001:2.0:1787828746.940880:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828746.940884:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828746.940888:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787828748.988895:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828748.988901:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828748.988906:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828750.012897:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828750.012902:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828750.012907:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828751.228803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828751.228815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828751.228818:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828751.228823:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828751.228840:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828751.228848:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2898
00000400:00000200:2.0:1787828751.228859:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828751.228866:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828751.228870:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828751.228873:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.228876:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828751.228878:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.228880:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828751.228884:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828751.228886:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828751.228904:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828751.228908:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828751.228910:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828751.228913:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828751.228916:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.228918:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828751.228920:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.228921:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828751.228924:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828751.228927:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828751.228930:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828751.228934:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828751.228938:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828751.228941:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.228943:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828751.228947:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828751.228960:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828751.228963:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828751.228968:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828751.228970:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.228985:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828751.228988:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828751.228994:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828751.229003:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828751.229011:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828751.229022:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828751.229033:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828751.229038:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828751.229042:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828751.229069:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828751.229072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828751.229076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828751.229078:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828751.229079:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828751.229082:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828751.229084:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2898
00000400:00000200:2.0:1787828751.229088:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828751.229091:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828751.229094:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828751.229098:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229101:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828751.229104:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787828751.229105:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229111:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828751.229113:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787828751.229114:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1787828751.229115:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828751.229119:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787828751.229120:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1787828751.229123:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828751.229169:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828751.229172:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828751.229173:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828751.229177:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229183:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828751.229187:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229193:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828751.229196:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787828751.229197:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:3.0:1787828751.229201:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828751.229203:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229207:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.229207:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787828751.229212:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828751.229212:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828751.229216:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828751.229220:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787828751.229221:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327275392:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787828751.229225:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.229229:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787828751.229230:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787828751.229236:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1787828751.229237:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787828751.229241:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787828751.229242:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.229247:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787828751.229248:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828751.229254:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
02000000:00000010:3.0:1787828751.229256:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
00000020:00000040:2.0:1787828751.229257:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828751.229261:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828751.229263:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828751.229265:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.229267:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828751.229270:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787828751.229270:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828751.229272:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828751.229276:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
02000000:00000010:2.0:1787828751.229279:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828751.229294:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828751.229296:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00100000:2.0:1787828751.229302:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787828751.229307:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275392, portal 25
00000100:00000001:2.0:1787828751.229312:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828751.229315:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828751.229318:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.229321:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828751.229325:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00000001:2.0:1787828751.229331:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828751.229336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828751.229340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:3.0:1787828751.229340:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828751.229343:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828751.229345:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828751.229350:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828751.229355:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2898
00000400:00000200:2.0:1787828751.229362:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828751.229364:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1787828751.229366:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828751.229380:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828751.229385:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:3.0:1787828751.229387:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229388:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828751.229393:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229395:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:3.0:1787828751.229396:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000020:00000001:2.0:1787828751.229397:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828751.229400:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828751.229403:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:3.0:1787828751.229405:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327275392, offset 0
00000020:00000040:2.0:1787828751.229406:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828751.229409:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828751.229413:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828751.229416:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229418:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787828751.229421:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787828751.229423:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229427:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828751.229430:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828751.229433:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229436:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.229437:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828751.229450:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828751.229453:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787828751.229456:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:2.0:1787828751.229457:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.229463:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828751.229466:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787828751.229467:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00000001:2.0:1787828751.229470:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.229473:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.229476:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000010:2.0:1787828751.229478:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000400:00000200:3.0:1787828751.229482:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:2.0:1787828751.229483:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828751.229488:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787828751.229489:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:2.0:1787828751.229490:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.229493:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828751.229496:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787828751.229497:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39923:39923:256:4294967295] 192.168.203.116@tcp LPNI seq info [39923:39923:8:4294967295]
02000000:00000001:2.0:1787828751.229499:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828751.229506:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000400:00000200:3.0:1787828751.229506:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828751.229511:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828751.229517:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828751.229525:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787828751.229538:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828751.229543:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787828751.229545:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787828751.229548:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828751.229567:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828751.229570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:3.0:1787828751.229570:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000100:00080000:2.0:1787828751.229577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828751.229578:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828751.229580:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828751.229582:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828751.229587:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2898
00000800:00000200:3.0:1787828751.229589:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:2.0:1787828751.229592:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828751.229599:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:3.0:1787828751.229600:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00080000:2.0:1787828751.229601:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828751.229608:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229611:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1787828751.229612:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828751.229615:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229617:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828751.229619:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828751.229620:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828751.229621:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828751.229623:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828751.229624:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828751.229626:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828751.229628:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229629:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828751.229634:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229637:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:3.1:1787828751.229638:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000020:00000040:2.0:1787828751.229639:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828751.229643:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229648:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828751.229650:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828751.229675:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828751.229678:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.229685:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828751.229686:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828751.229688:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828751.229690:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828751.229695:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828751.229698:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828751.229703:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828751.229705:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787828751.229708:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828751.229709:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828751.229713:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:3.0:1787828751.229717:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
02000000:00000001:2.0:1787828751.229718:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828751.229727:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828751.229732:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.229733:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:2.0:1787828751.229741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828751.229746:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787828751.229758:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787828751.229759:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000100:00000001:2.0:1787828751.229762:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828751.229764:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000040:2.0:1787828751.229770:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.229772:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828751.229781:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828751.229782:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828751.229790:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2909)
00000100:00000001:3.0:1787828751.229803:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.229807:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.229814:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.229816:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828751.229824:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.229828:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828751.229841:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297572013
00000100:00000010:3.0:1787828751.229846:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f120.
00000100:00000001:3.0:1787828751.229853:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828751.229865:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297572013
00000100:00000010:3.0:1787828751.229868:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fd20.
00000100:00000001:3.0:1787828751.229871:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828751.229873:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828751.229876:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.229878:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828751.229883:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828751/real 1787828751]  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828751.229892:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828751.229894:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828751.229896:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828751.229899:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828751.229900:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828751.229902:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828751.229905:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828751.229912:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.229921:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828751.229924:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.229928:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.229930:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828751.229936:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.229945:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828751.229951:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.229962:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828751.229967:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.229969:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.229971:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.229986:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828751.229988:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828751.229991:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828751.229995:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828751.229997:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828751.230001:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828751.230008:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327275392:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828751.230013:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230015:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.230017:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327275392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230022:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828751.230025:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828751.230027:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828751.230030:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828751.230032:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828751.230034:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828751.230036:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828751.230039:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828751.230043:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828751.230045:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828751.230047:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:3.0:1787828751.230050:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828751.230051:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230053:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230062:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828751.230063:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230066:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828751.230067:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828751.230068:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828751.230071:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230078:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828751.230079:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.230081:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828751.230082:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828751.230084:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828751.230086:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275520:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828751.230089:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828751.230091:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828751.230092:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.230094:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828751.230097:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828751.230098:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.230102:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828751.230104:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828751.230107:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828751.230110:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275520, portal 10
00000100:00000001:3.0:1787828751.230112:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828751.230138:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:3.0:1787828751.230146:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230152:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828751.230154:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828751.230156:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828751.230158:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327275520, offset 0
00000400:00000200:3.0:1787828751.230162:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828751.230166:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828751.230170:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828751.230172:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828751.230175:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828751.230177:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828751.230179:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828751.230181:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39924:39924:256:4294967295] 192.168.203.116@tcp LPNI seq info [39924:39924:8:4294967295]
00000400:00000200:3.0:1787828751.230185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828751.230191:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828751.230194:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828751.230197:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828751.230201:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828751.230203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828751.230207:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828751.230209:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787828751.230213:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828751.230214:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828751.230217:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828751.230220:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828751.230222:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828751.230224:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828751.230226:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828751.230229:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230236:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.230238:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230249:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230250:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828751.230253:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828751.230256:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828751.230259:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828751.230261:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297571339
00000100:00000001:3.0:1787828751.230264:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297572013
00000100:00000001:3.0:1787828751.230266:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297571339
00000100:00000001:3.0:1787828751.230268:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297572013
00000100:00000001:3.0:1787828751.230269:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828751.230271:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230272:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828751.230275:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828751/real 1787828751]  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828751.230287:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828751.230289:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828751.230291:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828751.230292:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828751.230294:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828751.230295:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828751.230298:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828751.230304:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230309:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828751.230310:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230312:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.230313:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828751.230315:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230322:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828751.230325:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230330:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828751.230332:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230333:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.230334:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230335:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828751.230337:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828751.230339:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828751.230342:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828751.230345:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828751.230349:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828751.230359:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275520:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828751.230363:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828751.230372:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230381:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828751.230383:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.230386:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828751.230389:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828751.230392:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828751.230396:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275648:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828751.230400:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828751.230402:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828751.230404:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.230406:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828751.230409:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828751.230411:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.230414:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828751.230417:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828751.230420:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828751.230422:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275648, portal 4
00000100:00000001:3.0:1787828751.230424:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828751.230425:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:3.0:1787828751.230430:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230435:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828751.230438:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828751.230439:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828751.230441:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327275648, offset 0
00000400:00000200:3.0:1787828751.230444:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828751.230447:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828751.230456:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828751.230459:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828751.230461:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828751.230463:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828751.230465:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828751.230468:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39925:39925:256:4294967295] 192.168.203.116@tcp LPNI seq info [39925:39925:8:4294967295]
00000400:00000200:3.0:1787828751.230471:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828751.230476:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828751.230480:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828751.230482:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828751.230486:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828751.230488:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828751.230491:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828751.230493:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787828751.230497:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828751.230499:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828751.230502:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828751.230504:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828751.230507:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828751.230508:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828751.230510:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828751.230513:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230519:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.230522:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230529:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230531:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828751.230534:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828751.230537:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828751.230541:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828751.230555:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297571346
00000100:00000001:3.0:1787828751.230558:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297572013
00000100:00000001:3.0:1787828751.230561:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297571346
00000100:00000001:3.0:1787828751.230564:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297572013
00000100:00000001:3.0:1787828751.230567:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828751.230569:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230572:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828751.230578:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828751/real 1787828751]  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828751.230589:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828751.230590:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828751.230593:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828751.230595:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828751.230596:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828751.230599:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828751.230601:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828751.230608:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230613:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828751.230614:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230615:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.230616:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828751.230619:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230625:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828751.230628:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230633:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828751.230635:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230636:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.230637:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230638:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828751.230640:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828751.230643:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828751.230646:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828751.230648:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828751.230674:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828751.230685:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275648:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828751.230695:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828751.230699:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230708:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828751.230710:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.230713:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828751.230715:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828751.230718:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828751.230721:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275776:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828751.230726:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828751.230728:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828751.230730:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828751.230733:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828751.230737:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828751.230740:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828751.230744:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828751.230748:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828751.230752:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828751.230756:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275776, portal 4
00000100:00000001:3.0:1787828751.230757:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828751.230759:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:3.0:1787828751.230765:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230770:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828751.230772:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828751.230774:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828751.230776:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327275776, offset 0
00000400:00000200:3.0:1787828751.230779:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828751.230782:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828751.230784:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828751.230787:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828751.230798:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828751.230800:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828751.230803:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828751.230806:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39926:39926:256:4294967295] 192.168.203.116@tcp LPNI seq info [39926:39926:8:4294967295]
00000400:00000200:3.0:1787828751.230809:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828751.230814:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828751.230818:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828751.230821:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828751.230824:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828751.230826:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787828751.230830:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828751.230832:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787828751.230835:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828751.230837:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f900.
00000400:00000200:3.0:1787828751.230840:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828751.230843:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828751.230846:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99cc0
00000400:00000010:3.0:1787828751.230847:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99cc0.
00000100:00000001:3.0:1787828751.230849:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828751.230852:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230870:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.230872:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230877:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230879:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828751.230881:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828751.230883:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828751.230886:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828751.230888:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297571352
00000100:00000001:3.0:1787828751.230890:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297572013
00000100:00000001:3.0:1787828751.230892:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297571352
00000100:00000001:3.0:1787828751.230894:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297572013
00000100:00000001:3.0:1787828751.230896:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828751.230898:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.230900:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828751.230903:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828751/real 1787828751]  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828751.230909:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828751.230910:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828751.230911:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828751.230912:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828751.230914:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828751.230916:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828751.230920:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828751.230931:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230939:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828751.230941:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230944:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.230945:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828751.230950:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230971:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828751.230976:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.230986:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828751.230988:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230989:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828751.230991:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.230992:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828751.230994:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828751.230996:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828751.230999:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828751.231001:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828751.231005:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828751.231010:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327275776:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828751.231013:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.231015:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.231017:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327275520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.231022:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828751.231024:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828751.231026:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828751.231028:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828751.231029:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828751.231031:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828751.231032:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828751.231034:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828751.231036:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828751.231038:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828751.231039:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:3.0:1787828751.231041:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828751.231043:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.231044:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.231047:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327275648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.231052:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828751.231053:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828751.231054:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828751.231056:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828751.231057:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828751.231058:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828751.231059:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828751.231061:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:3.0:1787828751.231063:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828751.231064:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828751.231065:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:3.0:1787828751.231067:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828751.231068:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.231069:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828751.231071:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327275776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828751.231075:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828751.231076:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828751.231077:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828751.231078:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828751.231079:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828751.231080:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828751.231081:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828751.231082:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:3.0:1787828751.231084:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828751.231085:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828751.231086:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:3.0:1787828751.231087:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828751.231088:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.231089:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828751.231092:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828751.231093:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828751.231095:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828751.231096:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828751.932776:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297572013, at 4297572032
00000100:00000010:3.0:1787828751.932789:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd20.
00000100:00000040:2.0:1787828751.932818:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297572013, at 4297572032
00000100:00000010:2.0:1787828751.932835:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f120.
00000020:00000001:2.0:1787828753.084845:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828753.084857:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828753.084861:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828755.132885:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828755.132892:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828755.132897:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828756.156859:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828756.156864:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828756.156867:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828756.349325:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828756.349338:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828756.349344:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828756.349360:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.349384:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828756.349388:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.349403:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828756.349408:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828756.349414:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828756.349422:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327275904:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828756.349435:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828756.349442:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828756.349448:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.349454:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828756.349478:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828756.349484:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828756.349497:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828756.349508:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828756.349553:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828756.349568:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327275904, portal 25
00000100:00000001:3.0:1787828756.349578:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828756.349595:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:3.0:1787828756.349647:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.349699:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828756.349704:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828756.349708:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99cc0.
00000100:00000200:3.0:1787828756.349712:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327275904, offset 0
00000400:00000200:3.0:1787828756.349731:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828756.349747:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.349758:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828756.349768:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828756.349779:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828756.349787:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828756.349792:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828756.349801:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39927:39927:256:4294967295] 192.168.203.116@tcp LPNI seq info [39927:39927:8:4294967295]
00000400:00000200:3.0:1787828756.349814:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.349834:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828756.349853:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828756.349862:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828756.349898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828756.349917:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828756.349953:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828756.349971:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828756.349981:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297577133
00000100:00000010:3.0:1787828756.350007:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fd20.
00000100:00000001:3.0:1787828756.350023:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828756.350039:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297577133
00000100:00000010:3.0:1787828756.350048:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f960.
00000100:00000001:3.0:1787828756.350069:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828756.350080:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828756.350087:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.350140:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.350144:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828756.350156:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828756.350160:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828756.350165:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828756.350168:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828756.350171:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828756.350181:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.350225:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828756.350230:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.350237:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828756.350245:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828756.350255:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828756.350268:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276032:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828756.350286:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828756.350293:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828756.350315:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.350321:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828756.350331:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828756.350337:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828756.350345:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828756.350350:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828756.350356:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99f68.
00000100:00000200:3.0:1787828756.350361:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276032, portal 10
00000100:00000001:3.0:1787828756.350367:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828756.350370:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:3.0:1787828756.350383:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.350401:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828756.350409:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828756.350417:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f994c8.
00000100:00000200:3.0:1787828756.350426:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327276032, offset 0
00000400:00000200:3.0:1787828756.350440:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828756.350453:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.350466:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828756.350475:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828756.350486:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828756.350492:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828756.350501:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828756.350512:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39928:39928:255:4294967295] 192.168.203.116@tcp LPNI seq info [39928:39928:7:4294967295]
00000400:00000200:3.0:1787828756.350530:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.350549:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828756.350563:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828756.350574:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828756.350589:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828756.350603:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828756.350613:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828756.350621:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828756.350633:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828756.350639:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297576459
00000100:00000001:3.0:1787828756.350655:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297577133
00000100:00000001:3.0:1787828756.350684:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297576459
00000100:00000001:3.0:1787828756.350687:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297577133
00000100:00000001:3.0:1787828756.350691:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828756.350694:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.350697:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.350700:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828756.350705:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828756.350706:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828756.350709:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828756.350711:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828756.350712:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828756.350718:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.350728:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828756.350730:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.350733:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828756.350735:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828756.350738:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828756.350743:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276160:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828756.350748:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828756.350751:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828756.350753:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.350755:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828756.350760:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828756.350763:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828756.350768:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828756.350771:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98dc0.
00000400:00000010:3.0:1787828756.350776:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828756.350780:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276160, portal 4
00000100:00000001:3.0:1787828756.350783:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828756.350785:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:3.0:1787828756.350793:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.350803:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828756.350807:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828756.350810:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99e58.
00000100:00000200:3.0:1787828756.350814:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327276160, offset 0
00000400:00000200:3.0:1787828756.350821:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828756.350826:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.350832:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828756.350836:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828756.350840:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828756.350843:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828756.350846:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828756.350850:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39929:39929:254:4294967295] 192.168.203.116@tcp LPNI seq info [39929:39929:6:4294967295]
00000400:00000200:3.0:1787828756.350856:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.350864:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828756.350870:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828756.350875:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f200.
00000800:00000200:3.0:1787828756.350880:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828756.350885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828756.350889:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828756.350892:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828756.350897:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828756.350899:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297576466
00000100:00000001:3.0:1787828756.350902:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297577133
00000100:00000001:3.0:1787828756.350905:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297576466
00000100:00000001:3.0:1787828756.350908:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297577133
00000100:00000001:3.0:1787828756.350911:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828756.350914:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.350917:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828756.350922:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.350940:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828756.350942:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.350946:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828756.350948:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828756.350951:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828756.350955:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276288:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828756.350959:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828756.350962:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828756.350964:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828756.350967:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828756.350972:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828756.350975:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828756.350979:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828756.350983:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828756.350988:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828756.350991:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276288, portal 4
00000100:00000001:3.0:1787828756.350994:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828756.350996:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:3.0:1787828756.351006:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828756.351013:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828756.351016:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828756.351019:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99c38.
00000100:00000200:3.0:1787828756.351022:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327276288, offset 0
00000400:00000200:3.0:1787828756.351027:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828756.351033:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.351038:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828756.351042:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828756.351045:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828756.351048:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828756.351051:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828756.351056:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39930:39930:253:4294967295] 192.168.203.116@tcp LPNI seq info [39930:39930:5:4294967295]
00000400:00000200:3.0:1787828756.351061:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828756.351069:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828756.351075:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828756.351080:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f100.
00000800:00000200:3.0:1787828756.351085:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828756.351108:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828756.351112:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828756.351116:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828756.351120:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828756.351122:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297576471
00000100:00000001:3.0:1787828756.351125:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297577133
00000100:00000001:3.0:1787828756.351128:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297576471
00000100:00000001:3.0:1787828756.351130:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297577133
00000100:00000001:3.0:1787828756.351133:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828756.351135:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.351138:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.351141:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828756.351145:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828756.351146:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828756.351149:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828756.351151:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828756.351153:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828756.351155:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828757.052713:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297577133, at 4297577152
00000100:00000040:2.0:1787828757.052719:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297577133, at 4297577152
00000100:00000010:3.0:1787828757.052732:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f960.
00000100:00000010:2.0:1787828757.052735:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd20.
00000020:00000001:2.0:1787828758.204830:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828758.204833:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828758.204836:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828759.228867:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828759.228872:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828759.228876:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828759.421239:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828759.421247:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828759.421256:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828759.421271:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828756/real 1787828759]  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828759.421308:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828759.421315:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828759.421324:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828759.421334:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99f68
00000400:00000010:3.0:1787828759.421338:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99f68.
00000100:00000001:3.0:1787828759.421343:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828759.421353:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828759.421406:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421422:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828759.421426:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421431:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.421435:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828759.421444:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421451:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828759.421458:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421466:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828759.421469:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421470:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.421471:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421474:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828759.421477:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828759.421480:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828759.421485:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828759.421487:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828759.421491:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828759.421505:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276032:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828759.421511:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828759.421515:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828756/real 1787828759]  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828759.421523:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828759.421524:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828759.421526:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828759.421529:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828759.421531:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828759.421534:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828759.421537:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828759.421546:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421554:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828759.421555:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421557:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.421558:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828759.421562:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421569:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828759.421573:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421580:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828759.421583:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421584:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.421586:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421588:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828759.421590:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828759.421592:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828759.421596:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828759.421598:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828759.421603:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828759.421617:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327275904:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828759.421623:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828759.421626:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828759.421629:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327276032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421636:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828759.421639:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828759.421644:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828759.421648:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828759.421651:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828759.421654:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828759.421682:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828759.421685:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:3.0:1787828759.421690:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828759.421693:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828759.421696:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:3.0:1787828759.421700:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828759.421701:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421703:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828759.421707:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327275904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421713:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828759.421715:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828759.421716:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828759.421719:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828759.421720:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828759.421722:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828759.421724:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828759.421726:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:3.0:1787828759.421728:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828759.421729:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828759.421730:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:3.0:1787828759.421733:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828759.421734:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421736:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828759.421738:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828759.421739:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828759.421740:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828759.421744:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828756/real 1787828759]  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828759.421752:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828759.421754:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828759.421757:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98dc0.
00000400:00000200:3.0:1787828759.421759:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828759.421761:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828759.421764:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828759.421768:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828759.421778:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421786:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828759.421788:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421790:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.421791:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828759.421797:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421804:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828759.421808:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.421816:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828759.421819:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421821:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.421822:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.421824:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828759.421826:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828759.421828:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828759.421832:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828759.421835:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828759.421839:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828759.421847:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276160:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828759.421852:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828759.421855:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828756/real 1787828759]  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828759.421862:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828759.421864:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828759.421866:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828759.421868:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828759.421870:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828759.422394:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828759.422403:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828759.422417:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.422426:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828759.422428:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422431:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.422432:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828759.422437:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.422445:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828759.422450:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.422457:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828759.422460:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422462:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828759.422463:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422465:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828759.422467:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828759.422470:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828759.422474:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828759.422477:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828759.422481:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828759.422490:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276288:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828759.422494:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422496:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828759.422499:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327276160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.422505:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828759.422507:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828759.422510:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828759.422513:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828759.422515:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828759.422517:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828759.422520:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828759.422521:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:3.0:1787828759.422524:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828759.422526:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828759.422527:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:3.0:1787828759.422530:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828759.422532:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422534:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828759.422537:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327276288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828759.422545:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828759.422546:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828759.422548:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828759.422550:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828759.422552:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828759.422553:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828759.422555:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828759.422556:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:3.0:1787828759.422559:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828759.422560:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828759.422561:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:3.0:1787828759.422563:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828759.422564:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422566:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828759.422572:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828759.422574:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828759.422577:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828759.422578:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828761.276971:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828761.276979:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828761.276985:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828761.469169:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828761.469185:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828761.469194:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828761.469225:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.469280:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828761.469290:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.469306:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828761.469318:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828761.469359:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828761.469387:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327276416:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828761.469405:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828761.469416:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828761.469425:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.469436:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828761.469458:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828761.469473:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828761.469500:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828761.469528:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828761.469550:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828761.469578:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276416, portal 25
00000100:00000001:3.0:1787828761.469595:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828761.469603:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:3.0:1787828761.469634:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.469807:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828761.469825:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828761.469872:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828761.469885:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327276416, offset 0
00000400:00000200:3.0:1787828761.469919:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828761.469964:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.469988:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828761.470006:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828761.470389:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828761.470413:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828761.470418:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828761.470423:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39931:39931:256:4294967295] 192.168.203.116@tcp LPNI seq info [39931:39931:8:4294967295]
00000400:00000200:3.0:1787828761.470431:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.470444:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828761.470457:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.470466:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828761.470476:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828761.470481:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828761.470490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828761.470495:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828761.470500:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.470504:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828761.470512:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828761.470518:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828761.470526:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828761.470529:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828761.470535:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828761.470543:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470557:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.470563:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470573:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.470576:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828761.470584:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828761.470590:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828761.470595:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297582253
00000100:00000010:3.0:1787828761.470601:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f960.
00000100:00000001:3.0:1787828761.470610:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828761.470648:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297582253
00000100:00000010:3.0:1787828761.470654:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f360.
00000100:00000001:3.0:1787828761.470698:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828761.470707:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828761.470712:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.470716:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828761.470724:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761]  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828761.470748:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828761.470751:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828761.470768:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828761.470778:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828761.470781:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828761.470785:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828761.470791:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828761.470799:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470806:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828761.470807:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.470810:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.470812:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828761.470815:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470839:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828761.470844:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470851:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828761.470854:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.470855:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.470857:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.470859:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828761.470860:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828761.470863:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828761.470867:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828761.470869:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828761.470873:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828761.470881:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327276416:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828761.470885:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.470888:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.470891:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327276416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470896:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828761.470899:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828761.470902:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828761.470905:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828761.470907:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828761.470908:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828761.470910:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828761.470913:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:3.0:1787828761.470916:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828761.470918:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828761.470919:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:3.0:1787828761.470927:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828761.470929:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.470931:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.470937:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828761.470939:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.470941:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828761.470943:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828761.470945:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828761.470949:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.470955:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828761.470957:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.470960:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828761.470962:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828761.470965:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828761.470967:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276544:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828761.470971:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828761.470975:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828761.470977:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.470979:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828761.470983:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828761.470985:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828761.470989:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828761.470992:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828761.470997:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828761.471002:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276544, portal 10
00000100:00000001:3.0:1787828761.471004:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828761.471006:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:3.0:1787828761.471013:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471019:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828761.471021:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828761.471023:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828761.471026:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327276544, offset 0
00000400:00000200:3.0:1787828761.471031:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828761.471037:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.471042:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828761.471047:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828761.471051:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828761.471083:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828761.471086:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828761.471089:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39932:39932:256:4294967295] 192.168.203.116@tcp LPNI seq info [39932:39932:8:4294967295]
00000400:00000200:3.0:1787828761.471104:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.471110:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828761.471115:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.471119:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828761.471123:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828761.471126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828761.471130:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828761.471133:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828761.471136:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.471138:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828761.471141:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828761.471145:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828761.471148:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828761.471149:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828761.471152:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828761.471155:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471173:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.471176:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471182:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.471183:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828761.471186:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828761.471190:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828761.471201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828761.471203:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297581580
00000100:00000001:3.0:1787828761.471206:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297582253
00000100:00000001:3.0:1787828761.471208:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297581580
00000100:00000001:3.0:1787828761.471210:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297582253
00000100:00000001:3.0:1787828761.471212:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828761.471214:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.471216:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828761.471219:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761]  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828761.471226:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828761.471227:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828761.471229:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828761.471232:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828761.471233:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828761.471235:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828761.471238:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828761.471244:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471249:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828761.471251:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471252:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.471253:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828761.471256:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471262:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828761.471265:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471271:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828761.471273:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471275:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.471276:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471277:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828761.471279:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828761.471282:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828761.471284:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828761.471286:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828761.471289:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828761.471295:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276544:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828761.471299:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828761.471302:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471308:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828761.471309:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.471311:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828761.471313:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828761.471315:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828761.471317:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276672:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828761.471319:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828761.471321:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828761.471323:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.471324:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828761.471327:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828761.471329:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828761.471332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828761.471334:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828761.471337:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828761.471340:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276672, portal 4
00000100:00000001:3.0:1787828761.471341:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828761.471343:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:3.0:1787828761.471348:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471354:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828761.471356:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828761.471358:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828761.471360:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327276672, offset 0
00000400:00000200:3.0:1787828761.471364:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828761.471367:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.471370:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828761.471372:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828761.471375:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828761.471377:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828761.471379:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828761.471382:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39933:39933:256:4294967295] 192.168.203.116@tcp LPNI seq info [39933:39933:8:4294967295]
00000400:00000200:3.0:1787828761.471385:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.471391:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828761.471394:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.471397:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828761.471400:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828761.471402:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828761.471406:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828761.471408:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828761.471412:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.471413:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828761.471416:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828761.471419:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828761.471422:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828761.471423:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828761.471425:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828761.471428:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471435:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.471438:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471443:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.471445:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828761.471447:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828761.471449:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828761.471451:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828761.471453:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297581587
00000100:00000001:3.0:1787828761.471455:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297582253
00000100:00000001:3.0:1787828761.471457:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297581587
00000100:00000001:3.0:1787828761.471459:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297582253
00000100:00000001:3.0:1787828761.471461:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828761.471463:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.471464:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828761.471468:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761]  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828761.471474:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828761.471475:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828761.471477:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828761.471480:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828761.471482:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828761.471484:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828761.471488:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828761.471517:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471526:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828761.471529:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471532:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.471534:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828761.471540:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471550:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828761.471556:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471566:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828761.471569:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471570:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.471572:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471574:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828761.471576:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828761.471578:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828761.471581:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828761.471583:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828761.471587:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828761.471593:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276672:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828761.471597:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828761.471600:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471606:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828761.471607:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.471609:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828761.471611:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828761.471613:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828761.471615:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276800:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828761.471618:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828761.471620:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828761.471622:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828761.471623:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828761.471627:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828761.471629:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828761.471632:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828761.471634:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828761.471637:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828761.471640:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276800, portal 4
00000100:00000001:3.0:1787828761.471642:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828761.471643:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:3.0:1787828761.471649:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471655:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828761.471657:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828761.471659:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828761.471661:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327276800, offset 0
00000400:00000200:3.0:1787828761.471665:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828761.471668:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.471692:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828761.471695:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828761.471698:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828761.471700:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828761.471702:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828761.471705:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39934:39934:256:4294967295] 192.168.203.116@tcp LPNI seq info [39934:39934:8:4294967295]
00000400:00000200:3.0:1787828761.471728:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828761.471737:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828761.471743:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.471747:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828761.471752:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828761.471756:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828761.471761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828761.471765:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828761.471770:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828761.471772:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828761.471793:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828761.471797:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828761.471800:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828761.471802:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828761.471804:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828761.471808:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471817:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.471820:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471833:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.471835:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828761.471838:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828761.471857:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828761.471860:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828761.471862:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297581593
00000100:00000001:3.0:1787828761.471864:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297582253
00000100:00000001:3.0:1787828761.471866:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297581593
00000100:00000001:3.0:1787828761.471868:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297582253
00000100:00000001:3.0:1787828761.471870:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828761.471871:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.471873:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828761.471876:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761]  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828761.471882:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828761.471884:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828761.471885:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828761.471887:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828761.471889:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828761.471890:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828761.471893:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828761.471900:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471905:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828761.471906:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471907:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.471909:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828761.471911:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471918:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828761.471921:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471927:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828761.471928:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471930:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828761.471931:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471932:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828761.471934:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828761.471936:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828761.471955:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828761.471957:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828761.471960:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828761.471967:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327276800:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828761.471970:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.471972:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.471974:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327276544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.471980:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828761.471981:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828761.471984:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828761.471986:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828761.471987:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828761.471989:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828761.471991:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828761.471992:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:3.0:1787828761.471995:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828761.471997:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828761.471998:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:3.0:1787828761.472001:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828761.472002:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.472004:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.472006:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327276672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.472012:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828761.472013:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828761.472015:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828761.472017:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828761.472018:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828761.472019:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828761.472020:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828761.472021:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:3.0:1787828761.472023:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828761.472024:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828761.472025:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:3.0:1787828761.472027:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828761.472028:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.472029:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828761.472032:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327276800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828761.472037:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828761.472038:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828761.472039:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828761.472041:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828761.472042:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828761.472043:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828761.472044:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828761.472045:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:3.0:1787828761.472047:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828761.472048:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828761.472049:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:3.0:1787828761.472051:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828761.472089:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.472091:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828761.472094:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828761.472095:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828761.472097:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828761.472099:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828762.172732:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297582253, at 4297582272
00000100:00000040:2.0:1787828762.172735:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297582253, at 4297582272
00000100:00000010:3.0:1787828762.172746:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f360.
00000100:00000010:2.0:1787828762.172748:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f960.
00000020:00000001:2.0:1787828762.300856:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828762.300861:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828762.300865:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828764.348790:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828764.348796:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828764.348801:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828765.372922:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828765.372927:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828765.372931:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828766.588758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828766.588774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828766.588777:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828766.588782:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828766.588788:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828766.588798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2914
00000400:00000200:2.0:1787828766.588809:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828766.588817:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828766.588822:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828766.588826:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.588830:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828766.588833:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.588835:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828766.588840:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828766.588843:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828766.588846:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828766.588851:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828766.588854:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828766.588858:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828766.588862:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.588864:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828766.588866:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.588869:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828766.588873:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828766.588877:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828766.588880:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.588884:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828766.588888:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828766.588892:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.588894:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828766.588897:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828766.588908:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787828766.588913:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828766.588918:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828766.588920:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.588924:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828766.588927:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828766.588932:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828766.588941:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828766.588949:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828766.588959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828766.588971:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828766.588976:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828766.588981:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828766.589006:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828766.589010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828766.589013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828766.589015:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828766.589029:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828766.589032:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828766.589035:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2914
00000400:00000200:2.0:1787828766.589039:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828766.589043:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828766.589046:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828766.589049:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589051:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828766.589054:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589056:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828766.589059:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828766.589060:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828766.589062:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828766.589065:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828766.589067:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828766.589070:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828766.589074:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589076:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828766.589079:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589081:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828766.589084:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828766.589089:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589092:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.589094:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828766.589100:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828766.589104:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.589105:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828766.589107:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828766.589114:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000008:00100000:2.0:1787828766.589115:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828766.589117:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00000001:2.0:1787828766.589122:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.589127:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828766.589132:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787828766.589137:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828766.589145:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:3.0:1787828766.589155:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787828766.589156:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787828766.589159:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.589161:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.589164:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787828766.589167:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828766.589169:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828766.589171:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828766.589173:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.589177:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828766.589180:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327276928:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828766.589184:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:2.0:1787828766.589185:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828766.589189:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828766.589192:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.589193:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.589196:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828766.589206:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
00000100:00100000:2.0:1787828766.589208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787828766.589210:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828766.589230:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828766.589232:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1787828766.589235:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000100:00000001:2.0:1787828766.589239:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828766.589243:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000040:2.0:1787828766.589250:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828766.589251:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327276928, portal 25
00000100:00000001:3.0:1787828766.589257:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828766.589259:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402430400 : -116696307121216 : ffff95dd861143c0)
00000100:00000040:3.0:1787828766.589274:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828766.589282:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.589284:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787828766.589288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787828766.589289:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828766.589295:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828766.589300:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327276928, offset 0
00000100:00080000:2.0:1787828766.589303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828766.589307:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787828766.589307:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787828766.589314:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828766.589320:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787828766.589322:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828766.589328:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2914
00000400:00000200:3.0:1787828766.589332:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828766.589337:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828766.589341:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828766.589344:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828766.589346:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828766.589349:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39935:39935:256:4294967295] 192.168.203.116@tcp LPNI seq info [39935:39935:8:4294967295]
00000400:00000200:3.0:1787828766.589352:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.589359:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828766.589363:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828766.589371:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000010:3.0:1787828766.589376:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000400:00000200:2.0:1787828766.589380:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828766.589386:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787828766.589394:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828766.589395:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589405:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787828766.589406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787828766.589409:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.589417:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828766.589425:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.1:1787828766.589426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787828766.589430:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:3.1:1787828766.589431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1787828766.589437:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:3.0:1787828766.589443:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787828766.589444:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000010:3.0:1787828766.589446:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000020:00000001:2.0:1787828766.589455:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:3.0:1787828766.589459:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:2.0:1787828766.589461:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:3.0:1787828766.589465:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828766.589476:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000100:00000040:2.0:1787828766.589477:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:3.0:1787828766.589480:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:2.0:1787828766.589487:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828766.589488:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787828766.589494:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:3.0:1787828766.589495:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828766.589504:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828766.589519:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:3.0:1787828766.589521:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1787828766.589536:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:3.0:1787828766.589537:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828766.589554:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.589559:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.589561:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828766.589562:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787828766.589567:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828766.589569:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.589578:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828766.589587:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828766.589589:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828766.589591:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297587372
00000008:00100000:2.0:1787828766.589593:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000010:3.0:1787828766.589598:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f360.
00000100:00000001:3.0:1787828766.589605:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000008:00000001:2.0:1787828766.589614:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.589616:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297587372
00000100:00000001:2.0:1787828766.589618:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828766.589624:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fc00.
00000100:00000010:2.0:1787828766.589627:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828766.589632:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:3.0:1787828766.589632:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:2.0:1787828766.589640:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787828766.589640:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828766.589642:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.589648:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787828766.589649:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.589651:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828766.589694:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828766.589700:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828766.589704:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.589705:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828766.589720:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766]  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787828766.589721:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828766.589783:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828766.589788:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787828766.589788:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828766.589791:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828766.589797:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828766.589803:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000100:00000040:2.0:1787828766.589804:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787828766.589816:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:2.0:1787828766.589819:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828766.589825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787828766.589827:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:2.0:1787828766.589831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828766.589835:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1787828766.589838:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828766.589839:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828766.589848:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828766.589855:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2914
00000400:00000200:2.0:1787828766.589871:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:3.0:1787828766.589872:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787828766.589884:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:3.0:1787828766.589885:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828766.589890:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787828766.589892:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787828766.589894:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.589897:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828766.589903:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00100000:3.0:1787828766.589905:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828766.589913:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828766.589919:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828766.589924:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:3.0:1787828766.589930:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828766.589931:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828766.589938:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828766.589943:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828766.589948:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828766.589954:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:3.0:1787828766.589962:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:2.0:1787828766.589963:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828766.589977:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:3.0:1787828766.589978:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.589988:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.589992:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828766.589993:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.590003:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828766.590003:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828766.590009:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1787828766.590010:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828766.590032:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00080000:3.0:1787828766.590034:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:2.0:1787828766.590036:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828766.590043:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00080000:2.0:1787828766.590044:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828766.590048:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1787828766.590050:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828766.590059:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787828766.590064:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787828766.590065:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828766.590075:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.590084:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828766.590090:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1787828766.590093:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327276928:192.168.203.116@tcp:250:kworker.0
00000008:00000001:2.0:1787828766.590097:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.590105:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590108:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590112:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:2.0:1787828766.590113:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828766.590121:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000040:3.0:1787828766.590122:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861143c0 x1874671327276928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787828766.590130:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828766.590135:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828766.590140:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787828766.590141:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787828766.590149:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828766.590150:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:2.0:1787828766.590154:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828766.590155:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828766.590164:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000010:2.0:1787828766.590165:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000020:00000001:3.0:1787828766.590167:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828766.590171:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000100:00000001:2.0:1787828766.590172:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828766.590175:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828766.590179:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
00000100:00100000:2.0:1787828766.590183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787828766.590185:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828766.590189:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828766.590194:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861143c0.
00000100:00000001:2.0:1787828766.590199:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828766.590201:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828766.590203:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590204:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590207:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828766.590209:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590217:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828766.590219:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590220:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590227:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787828766.590228:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2924)
00000100:00000001:3.0:1787828766.590232:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828766.590235:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828766.590240:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590247:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828766.590248:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.590252:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828766.590254:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828766.590256:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828766.590258:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277056:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828766.590261:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828766.590263:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828766.590264:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.590265:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828766.590268:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828766.590269:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828766.590273:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828766.590275:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828766.590277:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828766.590280:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277056, portal 10
00000100:00000001:3.0:1787828766.590281:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828766.590282:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:3.0:1787828766.590287:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590291:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828766.590293:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828766.590295:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828766.590296:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327277056, offset 0
00000400:00000200:3.0:1787828766.590299:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828766.590302:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.590305:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828766.590307:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828766.590309:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828766.590311:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828766.590313:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828766.590315:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39936:39936:256:4294967295] 192.168.203.116@tcp LPNI seq info [39936:39936:8:4294967295]
00000400:00000200:3.0:1787828766.590318:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.590323:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828766.590325:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828766.590327:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828766.590330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828766.590332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828766.590336:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828766.590338:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828766.590341:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828766.590342:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828766.590345:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828766.590348:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828766.590350:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828766.590351:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828766.590353:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828766.590355:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590361:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828766.590363:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590369:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590370:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828766.590374:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828766.590378:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828766.590381:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828766.590383:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297586699
00000100:00000001:3.0:1787828766.590386:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297587372
00000100:00000001:3.0:1787828766.590389:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297586699
00000100:00000001:3.0:1787828766.590391:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297587372
00000100:00000001:3.0:1787828766.590393:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828766.590395:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590398:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828766.590402:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766]  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828766.590411:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828766.590413:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828766.590415:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828766.590417:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828766.590419:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828766.590421:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828766.590424:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828766.590429:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590434:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828766.590435:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590436:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.590437:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828766.590440:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590444:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828766.590447:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590451:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828766.590452:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590454:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.590455:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590456:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828766.590458:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828766.590460:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828766.590462:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828766.590463:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828766.590466:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828766.590471:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277056:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828766.590474:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828766.590476:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590481:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828766.590481:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.590483:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828766.590484:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828766.590485:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828766.590487:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277184:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828766.590489:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828766.590491:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828766.590492:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.590493:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828766.590495:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828766.590497:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828766.590500:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828766.590501:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828766.590503:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828766.590506:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277184, portal 4
00000100:00000001:3.0:1787828766.590507:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828766.590508:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:3.0:1787828766.590512:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590516:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828766.590518:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828766.590519:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828766.590521:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327277184, offset 0
00000400:00000200:3.0:1787828766.590524:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828766.590526:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.590529:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828766.590531:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828766.590533:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828766.590534:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828766.590536:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828766.590538:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39937:39937:256:4294967295] 192.168.203.116@tcp LPNI seq info [39937:39937:8:4294967295]
00000400:00000200:3.0:1787828766.590541:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.590545:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828766.590548:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828766.590550:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828766.590553:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828766.590555:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828766.590558:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828766.590559:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828766.590574:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828766.590576:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828766.590578:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828766.590580:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828766.590583:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828766.590584:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828766.590586:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828766.590588:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590593:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828766.590596:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590600:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590601:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828766.590603:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828766.590605:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828766.590608:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828766.590609:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297586706
00000100:00000001:3.0:1787828766.590611:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297587372
00000100:00000001:3.0:1787828766.590612:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297586706
00000100:00000001:3.0:1787828766.590614:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297587372
00000100:00000001:3.0:1787828766.590615:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828766.590616:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590617:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828766.590620:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766]  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828766.590627:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828766.590628:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828766.590630:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828766.590633:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828766.590635:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828766.590637:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828766.590640:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828766.590650:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590680:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828766.590681:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590682:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.590683:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828766.590687:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590692:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828766.590694:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590699:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828766.590700:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590701:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.590702:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590703:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828766.590704:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828766.590706:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828766.590708:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828766.590709:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828766.590712:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828766.590716:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277184:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828766.590719:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828766.590722:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590726:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828766.590727:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.590729:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828766.590730:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828766.590731:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828766.590733:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277312:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828766.590735:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828766.590738:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828766.590740:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828766.590742:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828766.590745:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828766.590747:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828766.590751:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828766.590754:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828766.590757:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828766.590760:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277312, portal 4
00000100:00000001:3.0:1787828766.590762:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828766.590763:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:3.0:1787828766.590770:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590778:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828766.590780:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828766.590783:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828766.590785:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327277312, offset 0
00000400:00000200:3.0:1787828766.590790:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828766.590793:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.590795:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828766.590798:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828766.590799:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828766.590801:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828766.590803:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828766.590806:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39938:39938:256:4294967295] 192.168.203.116@tcp LPNI seq info [39938:39938:8:4294967295]
00000400:00000200:3.0:1787828766.590811:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828766.590817:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828766.590822:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828766.590826:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828766.590829:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828766.590832:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828766.590837:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828766.590840:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828766.590845:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828766.590847:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828766.590851:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828766.590854:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828766.590856:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828766.590858:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828766.590859:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828766.590862:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590868:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828766.590870:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590874:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590875:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828766.590877:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828766.590880:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828766.590882:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828766.590883:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297586712
00000100:00000001:3.0:1787828766.590885:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297587372
00000100:00000001:3.0:1787828766.590887:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297586712
00000100:00000001:3.0:1787828766.590888:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297587372
00000100:00000001:3.0:1787828766.590889:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828766.590891:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.590892:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828766.590894:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766]  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828766.590899:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828766.590900:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828766.590901:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828766.590903:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828766.590904:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828766.590906:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828766.590908:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828766.590913:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590917:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828766.590918:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590919:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.590920:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828766.590922:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590926:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828766.590928:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590932:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828766.590933:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590934:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828766.590935:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590936:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828766.590937:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828766.590939:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828766.590941:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828766.590943:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828766.590945:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828766.590950:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277312:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828766.590953:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590954:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828766.590956:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327277056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590961:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828766.590962:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828766.590963:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828766.590965:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828766.590966:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828766.590966:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828766.590968:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828766.590969:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828766.590971:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828766.590972:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828766.590973:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:3.0:1787828766.590974:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828766.590975:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590976:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828766.590978:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327277184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.590983:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828766.590983:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828766.590984:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828766.590985:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828766.590986:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828766.590987:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828766.590988:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828766.590989:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:3.0:1787828766.590990:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828766.590991:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828766.590992:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:3.0:1787828766.590993:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828766.590994:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.590995:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828766.590996:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327277312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828766.591000:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828766.591001:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828766.591002:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828766.591004:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828766.591004:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828766.591005:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828766.591007:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828766.591007:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828766.591009:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828766.591009:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828766.591010:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:3.0:1787828766.591011:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828766.591012:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.591013:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828766.591016:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828766.591032:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828766.591034:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828766.591035:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828767.292725:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297587372, at 4297587392
00000100:00000010:2.0:1787828767.292743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f360.
00000100:00000040:3.0:1787828767.292754:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297587372, at 4297587392
00000100:00000010:3.0:1787828767.292768:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fc00.
00000020:00000001:2.0:1787828767.420858:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828767.420865:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828767.420870:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828768.444829:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828768.444835:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828768.444839:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828770.492884:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828770.492888:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828770.492892:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828771.516877:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828771.516881:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828771.516884:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828771.708795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828771.708808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828771.708812:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828771.708818:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828771.708824:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828771.708834:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2919
00000400:00000200:2.0:1787828771.708845:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828771.708853:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828771.708857:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828771.708862:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.708865:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.708868:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.708871:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828771.708898:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828771.708901:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828771.708904:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828771.708909:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828771.708912:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828771.708915:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828771.708918:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.708920:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.708922:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.708925:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828771.708929:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828771.708934:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828771.708937:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828771.708942:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828771.708947:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828771.708951:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.708953:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828771.708957:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828771.708971:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828771.708976:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828771.709002:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828771.709004:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709010:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828771.709042:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828771.709049:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828771.709061:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828771.709070:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828771.709083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828771.709097:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828771.709102:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828771.709108:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828771.709136:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828771.709139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828771.709154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828771.709155:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828771.709157:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828771.709159:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828771.709162:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2919
00000400:00000200:2.0:1787828771.709166:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828771.709169:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828771.709172:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828771.709174:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709177:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.709180:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709182:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828771.709186:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787828771.709186:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787828771.709188:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828771.709192:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787828771.709193:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1787828771.709194:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828771.709198:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:3.0:1787828771.709198:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787828771.709203:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828771.709207:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709209:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.709211:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709221:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:3.0:1787828771.709221:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787828771.709226:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:3.0:1787828771.709236:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787828771.709238:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828771.709239:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.709246:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787828771.709248:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.709250:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828771.709252:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828771.709256:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828771.709260:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828771.709265:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828771.709267:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327277440:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828771.709268:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828771.709275:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828771.709276:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1787828771.709280:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709283:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.709284:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1787828771.709286:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828771.709290:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828771.709290:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828771.709297:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828771.709298:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787828771.709299:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709302:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828771.709304:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828771.709306:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828771.709308:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000010:2.0:1787828771.709311:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828771.709315:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709329:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.709354:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:2.0:1787828771.709359:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787828771.709363:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000100:00000001:2.0:1787828771.709377:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1787828771.709378:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000001:2.0:1787828771.709382:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828771.709389:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828771.709392:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277440, portal 25
00000100:00000001:3.0:1787828771.709401:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828771.709405:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00000001:2.0:1787828771.709412:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828771.709420:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828771.709421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828771.709433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828771.709438:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828771.709442:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828771.709452:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787828771.709453:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828771.709489:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2919
00000100:00000040:3.0:1787828771.709493:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828771.709498:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000400:00000200:2.0:1787828771.709499:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:3.0:1787828771.709506:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327277440, offset 0
00000400:00000200:2.0:1787828771.709508:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828771.709514:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828771.709519:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828771.709541:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709544:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.709550:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709553:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828771.709557:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828771.709559:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828771.709562:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828771.709565:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828771.709570:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:3.0:1787828771.709570:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787828771.709577:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828771.709585:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828771.709587:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828771.709590:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.709599:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828771.709601:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787828771.709604:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828771.709610:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:3.0:1787828771.709612:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:2.0:1787828771.709620:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:3.0:1787828771.709624:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787828771.709628:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.709633:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:2.0:1787828771.709635:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828771.709641:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787828771.709646:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39939:39939:256:4294967295] 192.168.203.116@tcp LPNI seq info [39939:39939:8:4294967295]
02000000:00000001:2.0:1787828771.709648:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709657:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787828771.709661:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:2.0:1787828771.709663:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828771.709669:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709675:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.709684:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000010:2.0:1787828771.709686:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828771.709694:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828771.709755:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1787828771.709756:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787828771.709757:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.709765:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828771.709768:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828771.709771:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787828771.709772:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
02000000:00000010:2.0:1787828771.709778:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828771.709790:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787828771.709799:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:2.0:1787828771.709800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1787828771.709812:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787828771.709813:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828771.709820:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828771.709828:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828771.709829:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787828771.709835:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828771.709867:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709869:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787828771.709873:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000100:00080000:2.0:1787828771.709875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828771.709881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828771.709883:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828771.709895:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787828771.709898:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787828771.709899:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828771.709910:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2919
00000400:00000200:3.0:1787828771.709914:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787828771.709922:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828771.709927:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828771.709930:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787828771.709933:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000100:00000040:2.0:1787828771.709935:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:3.0:1787828771.709939:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:2.0:1787828771.709941:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828771.709946:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.709948:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828771.709951:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828771.709953:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828771.709956:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787828771.709960:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1787828771.709961:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828771.709968:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:3.0:1787828771.709974:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787828771.709975:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828771.710010:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828771.710019:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828771.710021:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828771.710027:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:3.0:1787828771.710059:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828771.710061:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828771.710071:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:3.0:1787828771.710072:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.710078:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:2.0:1787828771.710082:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828771.710086:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.710088:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.710093:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828771.710094:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828771.710097:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297592493
02000000:00000001:2.0:1787828771.710101:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1787828771.710104:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fc00.
02000000:00000001:2.0:1787828771.710106:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828771.710110:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828771.710113:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828771.710114:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000008:00000001:2.0:1787828771.710119:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.710121:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297592493
00000100:00000001:2.0:1787828771.710123:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828771.710128:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f8a0.
00000100:00000010:2.0:1787828771.710131:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000100:00000001:3.0:1787828771.710133:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000040:2.0:1787828771.710135:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:3.0:1787828771.710137:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828771.710140:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787828771.710153:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828771.710156:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.710158:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787828771.710159:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828771.710162:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828771.710166:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828771.710166:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771]  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787828771.710172:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828771.710179:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787828771.710180:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828771.710184:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828771.710186:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000100:00100000:2.0:1787828771.710187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828771.710189:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828771.710193:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828771.710197:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787828771.710202:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1787828771.710204:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828771.710207:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828771.710216:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828771.710230:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828771.710232:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828771.710236:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2929)
00000100:00000001:3.0:1787828771.710244:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828771.710246:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710248:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.710249:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828771.710254:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710263:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828771.710269:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710277:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828771.710280:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710282:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.710284:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710287:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828771.710289:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828771.710293:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828771.710299:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828771.710302:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828771.710308:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828771.710345:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327277440:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828771.710351:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710353:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.710356:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327277440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710364:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828771.710367:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828771.710371:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828771.710375:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828771.710377:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828771.710379:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828771.710382:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828771.710385:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:3.0:1787828771.710390:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828771.710392:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828771.710394:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:3.0:1787828771.710398:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828771.710399:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710402:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710411:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828771.710413:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.710416:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828771.710418:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828771.710419:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828771.710423:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710431:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828771.710432:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.710435:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828771.710437:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828771.710439:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828771.710442:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277568:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828771.710446:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828771.710448:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828771.710450:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.710452:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828771.710473:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828771.710475:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.710479:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828771.710482:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828771.710486:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828771.710489:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277568, portal 10
00000100:00000001:3.0:1787828771.710491:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828771.710493:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:3.0:1787828771.710512:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710518:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828771.710520:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828771.710538:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828771.710541:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327277568, offset 0
00000400:00000200:3.0:1787828771.710545:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828771.710550:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828771.710554:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828771.710557:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828771.710560:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828771.710563:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828771.710566:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828771.710569:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39940:39940:256:4294967295] 192.168.203.116@tcp LPNI seq info [39940:39940:8:4294967295]
00000400:00000200:3.0:1787828771.710573:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828771.710580:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828771.710585:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828771.710589:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828771.710593:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828771.710597:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828771.710601:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828771.710604:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828771.710608:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828771.710610:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828771.710614:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828771.710618:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828771.710621:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828771.710623:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828771.710625:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828771.710629:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710644:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.710647:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710654:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.710656:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828771.710659:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828771.710663:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828771.710666:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828771.710669:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297591819
00000100:00000001:3.0:1787828771.710672:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297592493
00000100:00000001:3.0:1787828771.710675:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297591819
00000100:00000001:3.0:1787828771.710677:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297592493
00000100:00000001:3.0:1787828771.710679:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828771.710681:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.710683:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828771.710687:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771]  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828771.710696:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828771.710697:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828771.710699:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828771.710701:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828771.710703:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828771.710705:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828771.710708:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828771.710756:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710763:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828771.710765:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710767:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.710768:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828771.710772:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710787:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828771.710791:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710797:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828771.710799:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710801:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.710803:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.710805:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828771.710807:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828771.710810:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828771.710812:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828771.710814:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828771.710818:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828771.710825:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277568:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828771.710828:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828771.710831:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710838:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828771.710839:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.710841:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828771.710843:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828771.710846:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828771.710849:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277696:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828771.710852:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828771.710854:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828771.710856:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.710874:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828771.710878:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828771.710880:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.710884:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828771.710886:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828771.710890:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828771.710904:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277696, portal 4
00000100:00000001:3.0:1787828771.710906:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828771.710908:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:3.0:1787828771.710914:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.710921:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828771.710924:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828771.710926:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828771.710928:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327277696, offset 0
00000400:00000200:3.0:1787828771.710932:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828771.710936:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828771.710939:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828771.710942:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828771.710945:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828771.710947:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828771.710950:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828771.710953:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39941:39941:256:4294967295] 192.168.203.116@tcp LPNI seq info [39941:39941:8:4294967295]
00000400:00000200:3.0:1787828771.710957:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828771.710964:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828771.710968:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828771.710971:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828771.710976:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828771.710979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828771.711004:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828771.711007:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828771.711027:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828771.711029:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828771.711032:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828771.711043:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828771.711045:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828771.711047:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828771.711049:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828771.711052:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711058:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.711060:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711065:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.711067:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828771.711069:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828771.711072:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828771.711074:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828771.711076:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297591827
00000100:00000001:3.0:1787828771.711078:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297592493
00000100:00000001:3.0:1787828771.711080:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297591827
00000100:00000001:3.0:1787828771.711081:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297592493
00000100:00000001:3.0:1787828771.711083:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828771.711084:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.711085:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828771.711088:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771]  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828771.711094:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828771.711095:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828771.711097:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828771.711098:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828771.711099:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828771.711101:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828771.711103:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828771.711109:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711113:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828771.711114:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711115:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.711116:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828771.711119:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711123:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828771.711126:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711131:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828771.711133:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711134:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.711135:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711137:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828771.711138:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828771.711139:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828771.711141:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828771.711143:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828771.711145:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828771.711155:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277696:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828771.711158:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828771.711161:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711165:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828771.711166:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.711168:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828771.711169:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828771.711170:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828771.711172:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277824:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828771.711174:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828771.711175:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828771.711176:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828771.711178:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828771.711180:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828771.711181:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828771.711184:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828771.711186:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828771.711188:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828771.711190:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277824, portal 4
00000100:00000001:3.0:1787828771.711192:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828771.711193:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:3.0:1787828771.711197:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711202:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828771.711204:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828771.711205:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828771.711207:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327277824, offset 0
00000400:00000200:3.0:1787828771.711209:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828771.711212:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828771.711215:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828771.711217:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828771.711219:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828771.711221:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828771.711223:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828771.711225:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39942:39942:256:4294967295] 192.168.203.116@tcp LPNI seq info [39942:39942:8:4294967295]
00000400:00000200:3.0:1787828771.711229:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828771.711234:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828771.711237:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828771.711239:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828771.711242:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828771.711244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787828771.711247:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828771.711249:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787828771.711251:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828771.711253:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828771.711256:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828771.711260:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828771.711263:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828771.711266:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828771.711268:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828771.711273:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711282:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.711286:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711295:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.711297:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828771.711301:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828771.711304:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828771.711308:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828771.711312:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297591832
00000100:00000001:3.0:1787828771.711330:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297592493
00000100:00000001:3.0:1787828771.711333:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297591832
00000100:00000001:3.0:1787828771.711335:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297592493
00000100:00000001:3.0:1787828771.711338:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828771.711340:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.711342:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828771.711346:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771]  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828771.711353:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828771.711355:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828771.711357:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828771.711370:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828771.711372:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828771.711374:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828771.711377:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828771.711385:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711392:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828771.711393:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711394:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.711396:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828771.711399:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711405:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828771.711408:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711414:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828771.711417:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711419:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828771.711420:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711422:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828771.711424:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828771.711426:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828771.711429:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828771.711431:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828771.711434:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828771.711441:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327277824:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828771.711444:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711447:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.711451:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327277568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711460:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828771.711463:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828771.711465:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828771.711469:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828771.711472:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828771.711474:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828771.711477:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828771.711480:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828771.711495:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828771.711497:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828771.711500:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:3.0:1787828771.711522:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828771.711524:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711527:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.711533:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327277696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711560:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828771.711561:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828771.711563:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828771.711566:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828771.711567:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828771.711568:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828771.711570:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828771.711572:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828771.711574:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828771.711576:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828771.711577:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:3.0:1787828771.711579:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828771.711580:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711582:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828771.711585:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327277824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828771.711591:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828771.711593:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828771.711594:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828771.711597:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828771.711598:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828771.711599:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828771.711601:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828771.711603:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:3.0:1787828771.711605:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828771.711606:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828771.711607:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:3.0:1787828771.711609:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828771.711610:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711612:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828771.711616:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828771.711617:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828771.711619:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828771.711621:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828772.412721:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297592493, at 4297592512
00000100:00000040:3.0:1787828772.412721:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297592493, at 4297592512
00000100:00000010:3.0:1787828772.412740:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f8a0.
00000100:00000010:2.0:1787828772.412743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fc00.
00000020:00000001:2.0:1787828773.564908:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828773.564913:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828773.564917:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828774.588868:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828774.588872:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828774.588875:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828776.636932:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828776.636936:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828776.636939:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828776.829089:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828776.829096:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828776.829107:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828776.829117:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829134:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828776.829137:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.829144:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828776.829147:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828776.829151:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828776.829159:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327277952:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828776.829164:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828776.829170:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828776.829174:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.829178:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828776.829185:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828776.829189:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828776.829199:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828776.829204:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828776.829213:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828776.829221:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327277952, portal 25
00000100:00000001:3.0:1787828776.829228:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828776.829231:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:3.0:1787828776.829238:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829251:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828776.829255:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828776.829260:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828776.829264:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327277952, offset 0
00000400:00000200:3.0:1787828776.829275:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828776.829287:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.829297:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828776.829305:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828776.829312:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828776.829319:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828776.829325:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828776.829332:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39943:39943:256:4294967295] 192.168.203.116@tcp LPNI seq info [39943:39943:8:4294967295]
00000400:00000200:3.0:1787828776.829343:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.829359:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828776.829370:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.829376:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828776.829387:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828776.829392:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828776.829402:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828776.829407:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828776.829417:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.829420:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828776.829430:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828776.829440:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828776.829446:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828776.829452:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828776.829466:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828776.829473:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829491:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.829495:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829503:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.829506:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828776.829514:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828776.829522:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828776.829527:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297597612
00000100:00000010:3.0:1787828776.829533:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f8a0.
00000100:00000001:3.0:1787828776.829542:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828776.829551:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297597612
00000100:00000010:3.0:1787828776.829556:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f7e0.
00000100:00000001:3.0:1787828776.829563:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828776.829566:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828776.829569:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.829572:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828776.829578:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776]  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828776.829600:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828776.829602:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828776.829607:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828776.829610:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828776.829613:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828776.829616:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828776.829622:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828776.829645:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829698:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828776.829702:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.829705:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.829708:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828776.829717:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829728:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828776.829738:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829750:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828776.829756:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.829760:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.829763:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.829770:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828776.829773:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828776.829776:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828776.829781:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828776.829785:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828776.829791:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828776.829810:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327277952:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828776.829818:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.829823:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.829828:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327277952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.829857:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828776.829863:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828776.829867:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828776.829871:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828776.829875:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828776.829879:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828776.829883:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828776.829889:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:3.0:1787828776.829895:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828776.829898:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828776.829902:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:3.0:1787828776.829909:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828776.829912:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.829915:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.829923:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828776.829926:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.829931:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828776.829937:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828776.829973:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828776.829980:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830007:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828776.830009:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.830014:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828776.830017:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828776.830021:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828776.830028:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278080:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828776.830033:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828776.830036:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828776.830039:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.830040:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828776.830045:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828776.830049:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828776.830057:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828776.830060:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828776.830064:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828776.830068:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278080, portal 10
00000100:00000001:3.0:1787828776.830070:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828776.830073:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:3.0:1787828776.830081:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830087:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828776.830089:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828776.830090:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828776.830093:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327278080, offset 0
00000400:00000200:3.0:1787828776.830096:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828776.830100:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.830103:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828776.830105:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828776.830107:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828776.830109:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828776.830111:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828776.830113:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39944:39944:256:4294967295] 192.168.203.116@tcp LPNI seq info [39944:39944:8:4294967295]
00000400:00000200:3.0:1787828776.830116:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.830121:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828776.830125:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.830127:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828776.830130:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828776.830132:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828776.830135:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828776.830137:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828776.830140:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.830141:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828776.830144:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828776.830146:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828776.830149:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828776.830150:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828776.830152:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828776.830154:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830160:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.830162:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830167:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830168:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828776.830170:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828776.830174:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828776.830176:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828776.830178:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297596939
00000100:00000001:3.0:1787828776.830180:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297597612
00000100:00000001:3.0:1787828776.830181:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297596939
00000100:00000001:3.0:1787828776.830183:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297597612
00000100:00000001:3.0:1787828776.830184:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828776.830186:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830188:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828776.830190:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776]  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828776.830196:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828776.830196:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828776.830197:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828776.830199:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828776.830200:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828776.830201:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828776.830203:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828776.830210:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830213:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828776.830214:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830215:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.830216:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828776.830218:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830222:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828776.830225:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830229:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828776.830231:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830232:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.830233:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830234:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828776.830236:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828776.830238:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828776.830241:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828776.830242:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828776.830245:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828776.830250:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278080:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828776.830252:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828776.830254:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830258:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828776.830259:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.830260:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828776.830261:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828776.830263:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828776.830265:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278208:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828776.830266:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828776.830267:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828776.830268:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.830269:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828776.830272:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828776.830273:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828776.830275:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828776.830277:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828776.830279:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828776.830281:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278208, portal 4
00000100:00000001:3.0:1787828776.830282:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828776.830283:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:3.0:1787828776.830287:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830292:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828776.830294:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828776.830295:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828776.830297:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327278208, offset 0
00000400:00000200:3.0:1787828776.830300:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828776.830302:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.830304:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828776.830306:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828776.830308:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828776.830310:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828776.830311:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828776.830313:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39945:39945:256:4294967295] 192.168.203.116@tcp LPNI seq info [39945:39945:8:4294967295]
00000400:00000200:3.0:1787828776.830316:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.830320:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828776.830323:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.830325:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828776.830327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828776.830329:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828776.830331:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828776.830333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828776.830336:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.830337:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828776.830339:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828776.830341:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828776.830343:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828776.830344:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828776.830346:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828776.830348:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830353:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.830355:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830360:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830361:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828776.830363:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828776.830365:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828776.830367:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828776.830369:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297596946
00000100:00000001:3.0:1787828776.830371:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297597612
00000100:00000001:3.0:1787828776.830372:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297596946
00000100:00000001:3.0:1787828776.830374:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297597612
00000100:00000001:3.0:1787828776.830375:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828776.830376:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830377:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828776.830379:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776]  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828776.830384:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828776.830384:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828776.830385:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828776.830387:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828776.830388:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828776.830389:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828776.830391:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828776.830397:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830401:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828776.830401:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830402:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.830403:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828776.830405:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830409:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828776.830411:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830417:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828776.830418:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830419:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.830420:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830421:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828776.830422:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828776.830424:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828776.830426:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828776.830427:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828776.830429:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828776.830434:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278208:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828776.830436:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830437:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.830439:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327278080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830443:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828776.830444:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828776.830445:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828776.830447:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828776.830447:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828776.830448:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828776.830450:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828776.830451:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:3.0:1787828776.830452:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828776.830453:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828776.830454:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:3.0:1787828776.830457:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828776.830458:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830459:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.830461:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327278208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830466:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828776.830466:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828776.830467:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828776.830469:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828776.830469:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828776.830470:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828776.830471:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828776.830472:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:3.0:1787828776.830473:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828776.830474:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828776.830474:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:3.0:1787828776.830476:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828776.830477:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830478:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830480:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828776.830481:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830482:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828776.830483:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828776.830484:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828776.830486:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830490:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828776.830491:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.830492:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828776.830493:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828776.830494:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828776.830496:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828776.830498:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828776.830499:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828776.830500:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828776.830502:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828776.830504:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828776.830506:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828776.830507:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828776.830509:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828776.830511:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828776.830513:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278336, portal 4
00000100:00000001:3.0:1787828776.830514:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828776.830515:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:3.0:1787828776.830519:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830523:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828776.830525:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828776.830527:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828776.830528:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327278336, offset 0
00000400:00000200:3.0:1787828776.830530:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828776.830532:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.830534:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828776.830536:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828776.830537:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828776.830539:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828776.830541:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828776.830543:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39946:39946:256:4294967295] 192.168.203.116@tcp LPNI seq info [39946:39946:8:4294967295]
00000400:00000200:3.0:1787828776.830546:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828776.830550:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828776.830553:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.830554:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828776.830557:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828776.830558:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828776.830561:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828776.830563:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828776.830565:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828776.830566:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828776.830568:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828776.830571:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828776.830573:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828776.830574:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828776.830576:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828776.830578:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830583:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.830585:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830589:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830590:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828776.830592:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828776.830594:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828776.830595:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828776.830597:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297596951
00000100:00000001:3.0:1787828776.830598:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297597612
00000100:00000001:3.0:1787828776.830600:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297596951
00000100:00000001:3.0:1787828776.830601:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297597612
00000100:00000001:3.0:1787828776.830603:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828776.830604:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830605:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828776.830607:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776]  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828776.830612:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828776.830612:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828776.830613:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828776.830614:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828776.830615:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828776.830616:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828776.830618:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828776.830628:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830632:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828776.830633:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830634:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.830635:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828776.830637:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830641:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828776.830643:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830647:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828776.830648:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830649:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828776.830650:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830651:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828776.830652:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828776.830654:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828776.830656:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828776.830657:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828776.830659:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828776.830664:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828776.830666:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830668:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828776.830684:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327278336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828776.830694:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828776.830695:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828776.830696:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828776.830698:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828776.830699:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828776.830699:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828776.830701:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828776.830702:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:3.0:1787828776.830704:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828776.830705:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828776.830706:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:3.0:1787828776.830708:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828776.830708:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830710:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828776.830712:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828776.830713:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828776.830714:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828776.830715:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828777.532724:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297597612, at 4297597632
00000100:00000040:3.0:1787828777.532724:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297597612, at 4297597632
00000100:00000010:3.0:1787828777.532749:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f7e0.
00000100:00000010:2.0:1787828777.532753:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f8a0.
00000020:00000001:2.0:1787828777.660831:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828777.660836:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828777.660841:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828779.708904:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828779.708909:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828779.708914:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828780.732863:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828780.732867:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828780.732869:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828781.948985:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828781.948992:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828781.948997:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828781.949008:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949029:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828781.949033:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.949039:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828781.949042:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828781.949048:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828781.949057:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327278464:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828781.949064:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828781.949070:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828781.949076:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.949079:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828781.949085:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828781.949090:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828781.949097:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828781.949102:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828781.949111:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828781.949121:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278464, portal 25
00000100:00000001:3.0:1787828781.949124:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828781.949128:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:3.0:1787828781.949135:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949165:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828781.949173:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828781.949185:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828781.949191:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327278464, offset 0
00000400:00000200:3.0:1787828781.949203:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828781.949217:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.949229:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828781.949240:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828781.949248:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828781.949255:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828781.949262:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828781.949270:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39947:39947:256:4294967295] 192.168.203.116@tcp LPNI seq info [39947:39947:8:4294967295]
00000400:00000200:3.0:1787828781.949280:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.949297:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828781.949309:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.949317:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828781.949328:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828781.949337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828781.949347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828781.949351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828781.949361:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.949366:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828781.949378:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828781.949390:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828781.949396:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828781.949400:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828781.949407:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828781.949415:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949438:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.949444:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949464:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.949470:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828781.949482:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828781.949493:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828781.949501:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297602732
00000100:00000010:3.0:1787828781.949510:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f7e0.
00000100:00000001:3.0:1787828781.949522:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828781.949536:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297602732
00000100:00000010:3.0:1787828781.949543:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f8a0.
00000100:00000001:3.0:1787828781.949548:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828781.949555:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828781.949559:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.949563:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828781.949573:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781]  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828781.949592:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828781.949596:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828781.949600:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828781.949606:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828781.949608:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828781.949611:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828781.949615:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828781.949622:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949628:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828781.949630:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949632:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.949634:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828781.949637:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949643:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828781.949646:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949651:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828781.949653:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949674:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.949676:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949678:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828781.949679:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828781.949681:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828781.949684:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828781.949686:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828781.949689:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828781.949696:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327278464:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828781.949700:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949702:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.949704:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327278464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949714:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828781.949716:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828781.949719:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828781.949721:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828781.949723:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828781.949724:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828781.949725:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828781.949728:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:3.0:1787828781.949732:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828781.949734:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828781.949735:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:3.0:1787828781.949737:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828781.949738:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949740:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949747:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828781.949749:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.949750:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828781.949751:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828781.949752:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828781.949755:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949760:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828781.949761:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.949762:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828781.949763:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828781.949765:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828781.949767:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278592:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828781.949770:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828781.949771:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828781.949773:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.949774:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828781.949776:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828781.949777:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828781.949780:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828781.949782:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828781.949785:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828781.949787:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278592, portal 10
00000100:00000001:3.0:1787828781.949789:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828781.949790:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:3.0:1787828781.949795:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949800:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828781.949802:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828781.949803:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828781.949805:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327278592, offset 0
00000400:00000200:3.0:1787828781.949807:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828781.949810:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.949813:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828781.949815:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828781.949817:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828781.949819:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828781.949820:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828781.949823:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39948:39948:256:4294967295] 192.168.203.116@tcp LPNI seq info [39948:39948:8:4294967295]
00000400:00000200:3.0:1787828781.949825:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.949830:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828781.949833:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.949835:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828781.949838:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828781.949840:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828781.949844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828781.949845:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828781.949848:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.949849:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828781.949851:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828781.949854:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828781.949856:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828781.949858:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828781.949860:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828781.949862:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949867:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.949869:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949874:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.949875:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828781.949877:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828781.949881:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828781.949883:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828781.949884:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297602059
00000100:00000001:3.0:1787828781.949886:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297602732
00000100:00000001:3.0:1787828781.949888:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297602059
00000100:00000001:3.0:1787828781.949889:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297602732
00000100:00000001:3.0:1787828781.949891:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828781.949892:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.949893:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828781.949896:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781]  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828781.949900:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828781.949901:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828781.949902:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828781.949904:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828781.949905:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828781.949906:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828781.949908:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828781.949933:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949942:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828781.949949:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949951:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.949952:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828781.949957:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949962:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828781.949964:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949969:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828781.949970:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949971:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.949972:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.949973:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828781.949975:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828781.949976:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828781.949978:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828781.949980:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828781.949982:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828781.949987:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278592:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828781.949990:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828781.949993:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.949997:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828781.949998:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.950000:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828781.950001:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828781.950003:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828781.950005:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278720:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828781.950007:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828781.950009:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828781.950010:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.950011:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828781.950013:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828781.950015:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828781.950018:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828781.950020:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828781.950022:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828781.950024:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278720, portal 4
00000100:00000001:3.0:1787828781.950026:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828781.950027:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:3.0:1787828781.950031:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950036:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828781.950038:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828781.950040:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828781.950041:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327278720, offset 0
00000400:00000200:3.0:1787828781.950044:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828781.950047:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.950049:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828781.950051:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828781.950053:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828781.950054:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828781.950056:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828781.950058:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39949:39949:256:4294967295] 192.168.203.116@tcp LPNI seq info [39949:39949:8:4294967295]
00000400:00000200:3.0:1787828781.950061:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.950066:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828781.950070:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.950072:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828781.950075:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828781.950076:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828781.950080:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828781.950081:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828781.950100:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.950102:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828781.950104:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828781.950107:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828781.950109:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828781.950110:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828781.950112:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828781.950115:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950120:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.950123:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950127:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.950128:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828781.950130:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828781.950133:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828781.950135:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828781.950137:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297602066
00000100:00000001:3.0:1787828781.950139:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297602732
00000100:00000001:3.0:1787828781.950141:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297602066
00000100:00000001:3.0:1787828781.950142:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297602732
00000100:00000001:3.0:1787828781.950144:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828781.950145:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.950146:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828781.950148:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781]  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828781.950154:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828781.950155:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828781.950157:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828781.950158:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828781.950159:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828781.950160:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828781.950162:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828781.950167:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950171:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828781.950172:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950174:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.950175:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828781.950177:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950181:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828781.950183:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950187:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828781.950189:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950190:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.950191:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950192:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828781.950193:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828781.950194:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828781.950196:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828781.950197:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828781.950200:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828781.950206:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278720:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828781.950208:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828781.950210:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950214:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828781.950215:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.950216:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828781.950217:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828781.950219:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828781.950220:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828781.950222:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828781.950223:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828781.950224:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828781.950225:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828781.950227:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828781.950228:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828781.950230:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828781.950232:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828781.950233:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828781.950235:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278848, portal 4
00000100:00000001:3.0:1787828781.950237:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828781.950238:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:3.0:1787828781.950242:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950245:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828781.950247:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828781.950248:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828781.950250:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327278848, offset 0
00000400:00000200:3.0:1787828781.950253:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828781.950255:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.950257:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828781.950259:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828781.950261:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828781.950263:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828781.950264:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828781.950267:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39950:39950:256:4294967295] 192.168.203.116@tcp LPNI seq info [39950:39950:8:4294967295]
00000400:00000200:3.0:1787828781.950269:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828781.950273:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828781.950276:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.950278:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828781.950280:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828781.950282:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828781.950284:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828781.950286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828781.950289:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828781.950290:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828781.950292:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828781.950294:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828781.950296:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828781.950297:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828781.950298:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828781.950300:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950305:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.950307:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950312:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.950313:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828781.950315:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828781.950317:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828781.950318:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828781.950320:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297602071
00000100:00000001:3.0:1787828781.950322:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297602732
00000100:00000001:3.0:1787828781.950324:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297602071
00000100:00000001:3.0:1787828781.950325:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297602732
00000100:00000001:3.0:1787828781.950326:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828781.950327:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.950328:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828781.950331:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781]  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828781.950335:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828781.950336:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828781.950336:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828781.950338:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828781.950339:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828781.950340:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828781.950343:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828781.950348:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950352:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828781.950353:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950354:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.950355:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828781.950357:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950361:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828781.950363:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950367:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828781.950369:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950370:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828781.950371:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950372:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828781.950373:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828781.950375:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828781.950377:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828781.950378:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828781.950380:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828781.950385:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327278848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828781.950387:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950388:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.950390:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327278592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950394:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828781.950395:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828781.950396:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828781.950397:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828781.950398:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828781.950399:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828781.950400:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828781.950402:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:3.0:1787828781.950403:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828781.950404:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828781.950405:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:3.0:1787828781.950407:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828781.950408:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950409:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.950411:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327278720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950420:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828781.950421:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828781.950422:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828781.950424:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828781.950425:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828781.950425:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828781.950426:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828781.950427:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:3.0:1787828781.950429:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828781.950429:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828781.950430:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:3.0:1787828781.950431:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828781.950432:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950433:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828781.950435:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327278848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828781.950438:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828781.950439:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828781.950440:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828781.950441:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828781.950442:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828781.950442:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828781.950444:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828781.950444:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:3.0:1787828781.950446:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828781.950446:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828781.950447:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:3.0:1787828781.950448:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828781.950449:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950450:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828781.950453:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828781.950454:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828781.950455:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828781.950457:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828782.652727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297602732, at 4297602752
00000100:00000040:2.0:1787828782.652728:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297602732, at 4297602752
00000100:00000010:3.0:1787828782.652742:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f8a0.
00000100:00000010:2.0:1787828782.652744:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f7e0.
00000020:00000001:2.0:1787828782.780865:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828782.780868:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828782.780871:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828783.804813:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828783.804818:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828783.804822:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828785.852839:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828785.852843:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828785.852846:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828786.876891:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828786.876897:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828786.876901:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828787.068808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828787.068819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828787.068822:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828787.068827:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828787.068832:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828787.068841:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2934
00000400:00000200:2.0:1787828787.068851:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828787.068859:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828787.068862:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828787.068866:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.068868:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.068871:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.068874:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828787.068888:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828787.068890:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828787.068893:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828787.068897:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828787.068900:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828787.068903:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828787.068906:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.068908:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.068910:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.068912:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828787.068914:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828787.068918:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828787.068921:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828787.068923:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828787.068927:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828787.068930:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.068932:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828787.068935:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828787.068944:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828787.068947:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828787.068951:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828787.068953:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.068956:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828787.068958:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828787.068962:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828787.068970:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828787.068977:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828787.068986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828787.068997:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828787.069001:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828787.069006:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828787.069030:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828787.069032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828787.069035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828787.069036:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828787.069038:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828787.069040:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828787.069042:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2934
00000400:00000200:2.0:1787828787.069045:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828787.069047:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828787.069048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828787.069051:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069052:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.069054:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069056:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828787.069058:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828787.069059:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828787.069061:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828787.069062:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828787.069064:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828787.069066:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828787.069069:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069072:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.069075:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069078:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828787.069082:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:3.0:1787828787.069082:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828787.069108:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787828787.069109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828787.069111:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787828787.069117:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828787.069122:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787828787.069126:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787828787.069131:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828787.069140:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069147:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828787.069152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828787.069155:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:2.0:1787828787.069156:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069162:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.069162:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828787.069176:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
02000000:00000001:3.0:1787828787.069178:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787828787.069180:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828787.069184:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828787.069185:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828787.069186:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069197:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828787.069198:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828787.069199:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828787.069208:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828787.069224:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00100000:3.0:1787828787.069228:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327278976:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828787.069231:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.069234:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:2.0:1787828787.069240:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787828787.069241:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828787.069246:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.069255:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787828787.069263:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828787.069265:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828787.069270:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
00000100:00000040:2.0:1787828787.069271:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828787.069287:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.069292:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828787.069294:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828787.069298:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828787.069303:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787828787.069304:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828787.069309:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:3.0:1787828787.069311:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000100:00000001:2.0:1787828787.069313:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828787.069316:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2934
00000400:00000200:2.0:1787828787.069319:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828787.069321:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828787.069328:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828787.069330:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069332:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.069335:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:3.0:1787828787.069338:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000001:2.0:1787828787.069339:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828787.069345:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828787.069347:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828787.069350:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:3.0:1787828787.069351:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327278976, portal 25
00000020:00000040:2.0:1787828787.069354:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000001:3.0:1787828787.069356:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:2.0:1787828787.069359:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:3.0:1787828787.069362:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000020:00000001:2.0:1787828787.069366:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828787.069373:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828787.069376:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069385:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.069390:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069393:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828787.069395:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1787828787.069396:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828787.069404:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069409:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828787.069410:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787828787.069412:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828787.069418:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1787828787.069419:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
02000000:00000001:2.0:1787828787.069421:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069427:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787828787.069430:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327278976, offset 0
00000008:00100000:2.0:1787828787.069431:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828787.069440:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069442:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828787.069446:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000400:00000200:3.0:1787828787.069447:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787828787.069457:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828787.069461:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828787.069462:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069464:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828787.069466:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787828787.069467:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828787.069469:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828787.069475:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000400:00000200:3.0:1787828787.069480:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828787.069481:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.069495:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:2.0:1787828787.069497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828787.069507:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787828787.069522:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787828787.069522:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787828787.069529:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.069534:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:2.0:1787828787.069535:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828787.069548:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.069550:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39951:39951:256:4294967295] 192.168.203.116@tcp LPNI seq info [39951:39951:8:4294967295]
00000100:00080000:2.0:1787828787.069551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828787.069562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828787.069567:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787828787.069567:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828787.069571:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828787.069575:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828787.069581:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2934
00000400:00000200:3.0:1787828787.069592:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828787.069599:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828787.069608:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000010:3.0:1787828787.069612:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000400:00000200:2.0:1787828787.069618:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828787.069622:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787828787.069630:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828787.069633:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:3.0:1787828787.069639:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000001:2.0:1787828787.069640:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828787.069644:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069649:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828787.069652:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828787.069654:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000200:3.1:1787828787.069702:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787828787.069706:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828787.069710:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:3.1:1787828787.069711:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000020:00000001:2.0:1787828787.069713:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828787.069724:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828787.069728:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069734:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1787828787.069735:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787828787.069744:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:3.0:1787828787.069744:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000100:00000001:2.0:1787828787.069753:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828787.069757:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:3.0:1787828787.069759:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1787828787.069761:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828787.069766:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828787.069768:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787828787.069774:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1787828787.069775:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828787.069782:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069785:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787828787.069786:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000008:00100000:2.0:1787828787.069788:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1787828787.069790:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000008:00000001:2.0:1787828787.069794:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069798:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.069799:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1787828787.069807:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000100:00000200:3.0:1787828787.069809:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787828787.069814:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828787.069824:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828787.069828:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069834:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828787.069835:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787828787.069836:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787828787.069839:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828787.069844:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787828787.069846:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828787.069856:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.069861:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.069865:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:2.0:1787828787.069867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828787.069871:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828787.069889:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828787.069893:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.069895:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828787.069899:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297607853
00000100:00000040:2.0:1787828787.069900:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787828787.069909:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f8a0.
00000100:00000001:2.0:1787828787.069913:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828787.069919:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2944)
00000100:00000001:3.0:1787828787.069919:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828787.069939:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297607853
00000100:00000010:3.0:1787828787.069943:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fc00.
00000100:00000001:3.0:1787828787.069946:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828787.069949:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828787.069952:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.069954:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828787.069961:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828787/real 1787828787]  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828787.069969:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828787.069971:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828787.069972:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828787.069974:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828787.069976:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828787.069978:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828787.069980:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828787.069987:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.069991:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828787.069992:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.069994:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.069996:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828787.069998:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070003:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828787.070007:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070012:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828787.070014:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070015:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070017:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070020:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828787.070021:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828787.070024:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828787.070027:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828787.070030:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828787.070034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828787.070043:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327278976:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828787.070049:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070052:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070055:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327278976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070063:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828787.070066:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070070:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828787.070073:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828787.070076:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828787.070078:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828787.070081:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828787.070083:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828787.070087:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828787.070089:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828787.070091:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:3.0:1787828787.070094:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828787.070095:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070097:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070103:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828787.070105:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070107:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828787.070109:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828787.070110:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828787.070113:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070119:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828787.070120:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.070122:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828787.070124:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828787.070126:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828787.070127:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279104:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828787.070130:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828787.070132:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828787.070133:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.070134:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070136:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828787.070138:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.070142:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828787.070145:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828787.070150:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828787.070153:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279104, portal 10
00000100:00000001:3.0:1787828787.070154:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828787.070156:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:3.0:1787828787.070164:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070171:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828787.070174:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828787.070177:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828787.070179:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327279104, offset 0
00000400:00000200:3.0:1787828787.070184:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828787.070189:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828787.070192:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828787.070194:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828787.070196:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828787.070198:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828787.070200:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828787.070203:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39952:39952:256:4294967295] 192.168.203.116@tcp LPNI seq info [39952:39952:8:4294967295]
00000400:00000200:3.0:1787828787.070206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828787.070212:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828787.070215:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828787.070217:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828787.070220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828787.070223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828787.070227:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828787.070228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787828787.070231:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828787.070233:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828787.070235:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828787.070238:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828787.070240:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828787.070241:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828787.070243:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828787.070246:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070251:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070253:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070258:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070259:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828787.070261:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828787.070265:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828787.070267:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828787.070268:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297607179
00000100:00000001:3.0:1787828787.070271:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297607853
00000100:00000001:3.0:1787828787.070273:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297607179
00000100:00000001:3.0:1787828787.070275:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297607853
00000100:00000001:3.0:1787828787.070276:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828787.070278:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070280:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828787.070282:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828787/real 1787828787]  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828787.070288:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828787.070289:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828787.070290:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828787.070292:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828787.070293:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828787.070294:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828787.070296:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828787.070301:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070306:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828787.070307:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070308:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070309:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828787.070311:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070316:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828787.070319:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070326:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828787.070328:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070330:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070331:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070333:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828787.070335:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828787.070338:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828787.070342:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828787.070345:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828787.070350:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828787.070359:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279104:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828787.070364:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828787.070367:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070372:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828787.070374:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.070376:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828787.070377:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828787.070379:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828787.070380:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279232:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828787.070383:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828787.070385:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828787.070387:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.070389:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070392:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828787.070395:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.070399:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828787.070401:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828787.070405:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828787.070409:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279232, portal 4
00000100:00000001:3.0:1787828787.070411:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828787.070412:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:3.0:1787828787.070420:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070428:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828787.070431:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828787.070433:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828787.070435:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327279232, offset 0
00000400:00000200:3.0:1787828787.070438:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828787.070441:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828787.070444:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828787.070446:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828787.070448:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828787.070450:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828787.070452:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828787.070454:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39953:39953:256:4294967295] 192.168.203.116@tcp LPNI seq info [39953:39953:8:4294967295]
00000400:00000200:3.0:1787828787.070457:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828787.070461:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828787.070464:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828787.070466:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828787.070468:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828787.070470:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828787.070473:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828787.070475:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787828787.070478:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828787.070479:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828787.070482:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828787.070484:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828787.070486:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828787.070487:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828787.070489:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828787.070492:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070497:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070499:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070505:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070507:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828787.070510:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828787.070513:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828787.070517:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828787.070519:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297607186
00000100:00000001:3.0:1787828787.070521:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297607853
00000100:00000001:3.0:1787828787.070524:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297607186
00000100:00000001:3.0:1787828787.070527:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297607853
00000100:00000001:3.0:1787828787.070529:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828787.070531:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070533:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828787.070538:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828787/real 1787828787]  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828787.070547:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828787.070548:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828787.070550:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828787.070553:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828787.070554:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828787.070556:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828787.070558:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828787.070564:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070568:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828787.070569:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070570:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070571:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828787.070573:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070578:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828787.070580:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070586:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828787.070587:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070588:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070589:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070591:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828787.070591:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828787.070593:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828787.070595:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828787.070596:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828787.070599:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828787.070603:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279232:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828787.070606:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828787.070608:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070613:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828787.070615:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.070617:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828787.070619:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828787.070621:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828787.070624:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279360:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828787.070628:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828787.070630:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828787.070632:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828787.070634:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070637:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828787.070639:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828787.070643:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828787.070646:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828787.070650:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828787.070653:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279360, portal 4
00000100:00000001:3.0:1787828787.070677:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828787.070679:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:3.0:1787828787.070687:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070694:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828787.070697:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828787.070700:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828787.070703:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327279360, offset 0
00000400:00000200:3.0:1787828787.070708:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828787.070712:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828787.070716:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828787.070719:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828787.070721:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828787.070722:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828787.070724:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828787.070726:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39954:39954:256:4294967295] 192.168.203.116@tcp LPNI seq info [39954:39954:8:4294967295]
00000400:00000200:3.0:1787828787.070729:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828787.070734:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828787.070737:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828787.070740:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828787.070743:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828787.070745:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787828787.070748:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828787.070749:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787828787.070752:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828787.070753:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828787.070756:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828787.070759:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828787.070761:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828787.070762:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828787.070764:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828787.070767:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070772:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070775:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828812 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070779:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070780:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828787.070782:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828787.070785:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828787.070787:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828787.070789:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297607192
00000100:00000001:3.0:1787828787.070790:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297607853
00000100:00000001:3.0:1787828787.070792:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297607192
00000100:00000001:3.0:1787828787.070793:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297607853
00000100:00000001:3.0:1787828787.070795:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828787.070796:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070798:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828787.070800:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828787/real 1787828787]  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828787.070804:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828787.070805:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828787.070807:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828787.070808:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828787.070809:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828787.070811:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828787.070813:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828787.070819:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070827:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828787.070829:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070830:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070831:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828787.070833:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070837:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828787.070839:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070843:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828787.070845:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070846:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828787.070846:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070848:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828787.070849:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828787.070850:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828787.070852:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828787.070853:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828787.070856:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828787.070861:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279360:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828787.070864:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070865:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070867:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327279104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070871:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828787.070872:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070886:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828787.070889:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828787.070890:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828787.070891:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828787.070892:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828787.070894:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828787.070895:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828787.070896:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828787.070897:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:3.0:1787828787.070899:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828787.070900:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070901:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070904:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327279232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070908:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828787.070909:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070910:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828787.070912:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828787.070912:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828787.070913:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828787.070914:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828787.070915:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:3.0:1787828787.070916:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828787.070917:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828787.070918:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:3.0:1787828787.070919:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828787.070920:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070921:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828787.070923:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327279360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828812 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828787.070927:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828787.070927:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828787.070928:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828787.070929:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828787.070930:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828787.070931:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828787.070933:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828787.070933:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:3.0:1787828787.070935:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828787.070936:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828787.070937:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:3.0:1787828787.070939:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828787.070939:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070941:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828787.070943:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828787.070944:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828787.070945:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828787.070946:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828787.772782:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297607853, at 4297607872
00000100:00000040:3.0:1787828787.772782:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297607853, at 4297607872
00000100:00000010:2.0:1787828787.772796:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f8a0.
00000100:00000010:3.0:1787828787.772799:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fc00.
00000020:00000001:2.0:1787828788.924796:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828788.924800:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828788.924804:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828789.948835:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828789.948839:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828789.948842:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828791.996838:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828791.996857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828791.996860:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828792.189108:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828792.189119:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828792.189123:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828792.189141:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.189164:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828792.189172:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.189181:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828792.189185:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828792.189189:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828792.189197:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327279488:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828792.189213:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828792.189221:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828792.189226:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.189233:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828792.189248:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828792.189256:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828792.189273:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828792.189285:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828792.189307:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828792.189322:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279488, portal 25
00000100:00000001:3.0:1787828792.189335:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828792.189341:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:3.0:1787828792.189361:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.189399:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828792.189414:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828792.189420:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828792.189427:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327279488, offset 0
00000400:00000200:3.0:1787828792.189438:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828792.189456:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.189468:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828792.189483:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828792.189491:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828792.189503:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828792.189518:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828792.189531:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39955:39955:256:4294967295] 192.168.203.116@tcp LPNI seq info [39955:39955:8:4294967295]
00000400:00000200:3.0:1787828792.189550:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.189593:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828792.189623:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.189630:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828792.189647:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828792.189715:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828792.189730:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828792.189738:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787828792.189751:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.189754:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828792.189764:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828792.189780:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828792.189793:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828792.189797:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828792.189809:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828792.189819:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.189834:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.189837:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.189862:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.189864:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828792.189871:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828792.189875:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828792.189893:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297612973
00000100:00000010:3.0:1787828792.189922:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fc00.
00000100:00000001:3.0:1787828792.189930:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828792.189935:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297612973
00000100:00000010:3.0:1787828792.189937:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f360.
00000100:00000001:3.0:1787828792.189940:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828792.189942:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828792.189956:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.189959:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828792.189963:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828792/real 1787828792]  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828792.189972:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828792.189975:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828792.189978:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828792.189981:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828792.189983:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828792.189986:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828792.189990:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828792.190000:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190008:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828792.190023:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190026:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190028:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828792.190033:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190039:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828792.190043:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190049:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828792.190051:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190052:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190053:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190055:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828792.190056:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828792.190059:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828792.190062:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828792.190064:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828792.190067:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828792.190073:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327279488:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828792.190077:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190079:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190082:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327279488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190086:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828792.190088:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190091:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828792.190093:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828792.190095:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828792.190097:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828792.190099:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828792.190102:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:3.0:1787828792.190106:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828792.190108:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828792.190110:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:3.0:1787828792.190112:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828792.190113:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190115:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190122:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828792.190123:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190125:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828792.190127:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828792.190128:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828792.190131:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190136:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828792.190137:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.190139:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828792.190141:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828792.190142:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828792.190144:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279616:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828792.190146:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828792.190148:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828792.190149:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.190150:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190152:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828792.190153:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828792.190156:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828792.190158:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828792.190161:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828792.190163:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279616, portal 10
00000100:00000001:3.0:1787828792.190165:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828792.190166:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:3.0:1787828792.190172:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190177:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828792.190179:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828792.190180:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828792.190182:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327279616, offset 0
00000400:00000200:3.0:1787828792.190185:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828792.190188:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.190191:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828792.190193:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828792.190195:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828792.190197:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828792.190199:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828792.190201:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39956:39956:256:4294967295] 192.168.203.116@tcp LPNI seq info [39956:39956:8:4294967295]
00000400:00000200:3.0:1787828792.190204:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.190209:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828792.190212:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.190215:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828792.190217:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828792.190220:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828792.190223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828792.190225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787828792.190228:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.190229:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828792.190232:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828792.190234:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828792.190237:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828792.190238:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828792.190241:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828792.190245:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190253:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190256:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190271:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190273:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828792.190277:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828792.190281:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828792.190285:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828792.190288:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297612299
00000100:00000001:3.0:1787828792.190291:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297612973
00000100:00000001:3.0:1787828792.190293:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297612299
00000100:00000001:3.0:1787828792.190294:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297612973
00000100:00000001:3.0:1787828792.190296:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828792.190298:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190299:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828792.190302:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828792/real 1787828792]  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828792.190307:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828792.190308:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828792.190310:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828792.190311:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828792.190313:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828792.190314:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828792.190316:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828792.190322:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190327:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828792.190328:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190329:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190330:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828792.190332:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190337:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828792.190339:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190344:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828792.190345:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190346:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190347:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190348:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828792.190350:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828792.190351:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828792.190354:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828792.190355:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828792.190357:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828792.190362:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279616:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828792.190365:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828792.190367:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190372:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828792.190373:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.190375:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828792.190376:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828792.190377:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828792.190379:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279744:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828792.190381:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828792.190383:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828792.190385:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.190386:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190389:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828792.190390:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828792.190393:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828792.190395:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828792.190397:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828792.190399:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279744, portal 4
00000100:00000001:3.0:1787828792.190400:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828792.190402:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:3.0:1787828792.190406:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190410:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828792.190412:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828792.190414:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828792.190415:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327279744, offset 0
00000400:00000200:3.0:1787828792.190418:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828792.190420:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.190422:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828792.190424:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828792.190426:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828792.190428:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828792.190430:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828792.190432:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39957:39957:256:4294967295] 192.168.203.116@tcp LPNI seq info [39957:39957:8:4294967295]
00000400:00000200:3.0:1787828792.190435:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.190439:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828792.190442:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.190444:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828792.190447:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828792.190449:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828792.190452:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828792.190454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787828792.190457:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.190459:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828792.190461:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828792.190463:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828792.190465:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828792.190467:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828792.190468:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828792.190470:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190476:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190478:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190483:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190484:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828792.190486:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828792.190488:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828792.190490:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828792.190491:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297612306
00000100:00000001:3.0:1787828792.190492:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297612973
00000100:00000001:3.0:1787828792.190494:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297612306
00000100:00000001:3.0:1787828792.190496:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297612973
00000100:00000001:3.0:1787828792.190497:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828792.190499:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190500:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828792.190502:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828792/real 1787828792]  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828792.190506:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828792.190507:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828792.190509:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828792.190510:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828792.190511:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828792.190513:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828792.190515:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828792.190521:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190525:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828792.190526:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190527:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190527:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828792.190530:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190534:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828792.190536:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190540:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828792.190541:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190542:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190543:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190544:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828792.190545:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828792.190546:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828792.190548:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828792.190550:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828792.190552:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828792.190559:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279744:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828792.190563:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828792.190567:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190574:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828792.190576:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.190579:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828792.190581:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828792.190583:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828792.190586:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279872:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828792.190590:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828792.190592:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828792.190594:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828792.190596:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190600:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828792.190602:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828792.190605:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828792.190608:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828792.190612:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828792.190614:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327279872, portal 4
00000100:00000001:3.0:1787828792.190615:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828792.190617:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:3.0:1787828792.190622:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190626:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828792.190628:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828792.190629:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828792.190631:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327279872, offset 0
00000400:00000200:3.0:1787828792.190633:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828792.190636:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.190638:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828792.190640:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828792.190642:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828792.190644:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828792.190646:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828792.190648:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39958:39958:256:4294967295] 192.168.203.116@tcp LPNI seq info [39958:39958:8:4294967295]
00000400:00000200:3.0:1787828792.190651:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828792.190674:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828792.190678:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.190680:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828792.190683:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828792.190685:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787828792.190688:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828792.190690:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787828792.190692:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828792.190694:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828792.190697:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828792.190701:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828792.190704:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828792.190706:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828792.190709:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828792.190713:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190721:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190724:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190732:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190734:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828792.190738:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828792.190741:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828792.190744:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828792.190746:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297612312
00000100:00000001:3.0:1787828792.190761:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297612973
00000100:00000001:3.0:1787828792.190762:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297612312
00000100:00000001:3.0:1787828792.190764:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297612973
00000100:00000001:3.0:1787828792.190766:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828792.190768:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190769:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828792.190772:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828792/real 1787828792]  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828792.190779:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828792.190780:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828792.190781:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828792.190783:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828792.190784:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828792.190785:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828792.190787:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828792.190793:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190798:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828792.190798:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190800:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190800:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828792.190803:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190807:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828792.190809:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190813:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828792.190814:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190815:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828792.190816:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190817:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828792.190818:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828792.190820:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828792.190823:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828792.190824:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828792.190827:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828792.190832:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327279872:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828792.190834:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190836:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190837:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327279616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190856:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828792.190857:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190859:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828792.190861:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828792.190873:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828792.190874:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828792.190875:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828792.190877:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:3.0:1787828792.190879:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828792.190880:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828792.190881:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:3.0:1787828792.190882:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828792.190883:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190885:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190887:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327279744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190891:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828792.190892:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190893:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828792.190895:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828792.190896:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828792.190897:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828792.190898:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828792.190899:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:3.0:1787828792.190901:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828792.190902:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828792.190903:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:3.0:1787828792.190905:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828792.190906:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190907:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828792.190909:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327279872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828792.190913:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828792.190913:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828792.190914:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828792.190915:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828792.190916:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828792.190917:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828792.190918:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828792.190918:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:3.0:1787828792.190920:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828792.190920:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828792.190921:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:3.0:1787828792.190922:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828792.190923:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190924:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828792.190926:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828792.190927:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828792.190929:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828792.190937:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828792.892826:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297612973, at 4297612992
00000100:00000040:3.0:1787828792.892831:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297612973, at 4297612992
00000100:00000010:2.0:1787828792.892852:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fc00.
00000100:00000010:3.0:1787828792.892861:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f360.
00000020:00000001:2.0:1787828793.020832:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828793.020849:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828793.020853:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828795.068993:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828795.069000:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828795.069005:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828796.092896:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828796.092901:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828796.092906:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828797.308827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828797.308838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828797.308841:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828797.308845:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828797.308858:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828797.308865:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2944
00000400:00000200:2.0:1787828797.308874:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828797.308880:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828797.308883:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828797.308886:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.308889:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828797.308892:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.308894:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828797.308898:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828797.308900:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828797.308902:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828797.308907:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828797.308909:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828797.308912:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828797.308914:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.308916:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828797.308919:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.308921:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828797.308924:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828797.308927:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828797.308930:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828797.308933:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828797.308937:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828797.308940:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.308942:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828797.308944:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828797.308954:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828797.308957:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828797.308962:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828797.308964:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.308967:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828797.308971:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828797.308975:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828797.308983:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828797.308990:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828797.309000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828797.309011:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828797.309016:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828797.309021:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828797.309053:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828797.309056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828797.309060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828797.309062:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828797.309064:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828797.309067:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828797.309070:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2944
00000400:00000200:2.0:1787828797.309074:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828797.309078:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828797.309081:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828797.309085:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309088:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828797.309088:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787828797.309093:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828797.309094:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828797.309097:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828797.309098:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787828797.309103:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828797.309105:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828797.309107:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1787828797.309119:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787828797.309120:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828797.309125:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828797.309130:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828797.309137:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828797.309137:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828797.309140:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309142:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787828797.309144:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1787828797.309146:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:3.0:1787828797.309147:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828797.309150:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:3.0:1787828797.309151:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787828797.309154:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00100000:3.0:1787828797.309158:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327280000:192.168.203.116@tcp:250:kworker.0
00000100:00080000:2.0:1787828797.309162:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828797.309164:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787828797.309169:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.309170:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787828797.309172:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828797.309175:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828797.309178:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828797.309180:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787828797.309181:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309185:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828797.309188:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000010:3.0:1787828797.309189:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
00000008:00000001:2.0:1787828797.309191:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309195:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.309196:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828797.309206:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828797.309209:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828797.309210:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828797.309211:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309216:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787828797.309216:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828797.309218:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828797.309221:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828797.309225:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
02000000:00000010:2.0:1787828797.309226:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828797.309232:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828797.309236:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00100000:2.0:1787828797.309237:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787828797.309245:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280000, portal 25
00000100:00000001:2.0:1787828797.309246:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828797.309249:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.309263:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1787828797.309264:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309268:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787828797.309277:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828797.309280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787828797.309283:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828797.309284:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828797.309286:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828797.309288:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828797.309292:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828797.309296:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2944
00000400:00000200:2.0:1787828797.309304:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828797.309308:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1787828797.309309:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828797.309313:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828797.309316:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787828797.309318:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309321:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1787828797.309322:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000040:2.0:1787828797.309324:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309339:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:3.0:1787828797.309339:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327280000, offset 0
00000020:00000001:2.0:1787828797.309345:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828797.309346:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828797.309347:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828797.309349:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:3.0:1787828797.309351:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787828797.309353:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828797.309356:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828797.309359:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309360:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828797.309362:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828797.309364:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828797.309365:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828797.309369:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828797.309373:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:3.0:1787828797.309374:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828797.309377:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828797.309379:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787828797.309381:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1787828797.309382:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828797.309385:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828797.309387:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787828797.309388:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828797.309392:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787828797.309394:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000008:00000001:2.0:1787828797.309395:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309399:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828797.309399:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000010:2.0:1787828797.309404:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000400:00000200:3.0:1787828797.309405:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39959:39959:256:4294967295] 192.168.203.116@tcp LPNI seq info [39959:39959:8:4294967295]
00000020:00000040:2.0:1787828797.309408:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:3.0:1787828797.309412:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828797.309418:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828797.309421:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309425:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787828797.309426:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1787828797.309428:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828797.309430:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828797.309438:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000800:00000200:3.0:1787828797.309439:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309444:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787828797.309446:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000100:00100000:2.0:1787828797.309451:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1787828797.309458:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828797.309464:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787828797.309472:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787828797.309474:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828797.309475:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828797.309479:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828797.309482:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:3.0:1787828797.309487:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.309491:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000100:00000001:2.0:1787828797.309495:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828797.309498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787828797.309500:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1787828797.309501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828797.309505:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828797.309506:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787828797.309509:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787828797.309510:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828797.309514:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2944
00000400:00000200:3.0:1787828797.309515:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000200:2.0:1787828797.309519:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828797.309520:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000400:00000200:2.0:1787828797.309524:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:3.0:1787828797.309526:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1787828797.309527:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828797.309533:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:3.0:1787828797.309534:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828797.309535:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828797.309539:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309543:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828797.309547:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828797.309550:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828797.309552:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828797.309555:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828797.309557:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828797.309559:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:3.0:1787828797.309560:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828797.309561:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309564:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828797.309568:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309570:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:3.0:1787828797.309570:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787828797.309574:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828797.309578:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828797.309581:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828797.309584:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828797.309587:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828797.309591:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309594:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828797.309594:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787828797.309595:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828797.309599:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000008:00000001:2.0:1787828797.309600:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828797.309603:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828797.309606:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828797.309607:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787828797.309608:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828797.309614:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828797.309616:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828797.309617:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828797.309618:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828797.309621:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787828797.309621:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297618092
02000000:00000001:2.0:1787828797.309623:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828797.309629:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f360.
02000000:00000010:2.0:1787828797.309630:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828797.309635:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.309639:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00100000:2.0:1787828797.309641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828797.309675:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828797.309677:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828797.309680:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309684:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297618093
00000100:00000001:2.0:1787828797.309687:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828797.309690:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f960.
00000100:00000040:2.0:1787828797.309692:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2955)
00000100:00000001:3.0:1787828797.309695:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828797.309700:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828797.309702:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.309705:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828797.309710:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828797/real 1787828797]  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828797.309733:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828797.309734:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828797.309736:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828797.309738:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828797.309739:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828797.309741:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828797.309744:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828797.309749:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309754:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828797.309755:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.309757:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.309758:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828797.309760:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309765:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828797.309768:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309773:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828797.309775:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.309776:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.309776:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.309778:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828797.309780:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828797.309781:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828797.309784:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828797.309786:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828797.309788:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828797.309794:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327280000:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828797.309798:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.309799:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.309801:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327280000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309819:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828797.309821:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828797.309822:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828797.309825:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828797.309826:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828797.309827:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828797.309829:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828797.309831:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828797.309834:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828797.309835:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828797.309836:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:3.0:1787828797.309839:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828797.309840:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.309841:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.309847:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828797.309848:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.309849:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828797.309851:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828797.309852:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828797.309855:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309859:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828797.309860:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.309862:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828797.309863:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828797.309864:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828797.309866:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280128:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828797.309869:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828797.309871:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828797.309872:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.309873:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828797.309875:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828797.309876:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828797.309879:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828797.309881:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828797.309883:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828797.309886:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280128, portal 10
00000100:00000001:3.0:1787828797.309887:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828797.309888:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:3.0:1787828797.309893:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309897:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828797.309899:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828797.309900:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828797.309902:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327280128, offset 0
00000400:00000200:3.0:1787828797.309904:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828797.309907:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828797.309909:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828797.309911:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828797.309913:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828797.309915:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828797.309917:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828797.309919:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39960:39960:256:4294967295] 192.168.203.116@tcp LPNI seq info [39960:39960:8:4294967295]
00000400:00000200:3.0:1787828797.309922:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828797.309926:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828797.309929:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.309931:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828797.309934:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828797.309936:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828797.309940:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828797.309941:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828797.309944:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.309945:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828797.309947:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828797.309950:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828797.309952:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828797.309953:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828797.309955:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828797.309958:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309963:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.309965:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.309969:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.309970:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828797.309972:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828797.309974:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828797.309976:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828797.309978:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297617419
00000100:00000001:3.0:1787828797.309980:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297618092
00000100:00000001:3.0:1787828797.309982:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297617419
00000100:00000001:3.0:1787828797.309983:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297618093
00000100:00000001:3.0:1787828797.309984:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828797.309985:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.309987:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828797.309989:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828797/real 1787828797]  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828797.309994:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828797.309994:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828797.309996:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828797.309997:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828797.309998:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828797.310000:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828797.310002:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828797.310007:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310011:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828797.310012:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310013:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.310014:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828797.310016:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310020:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828797.310022:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310026:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828797.310028:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310029:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.310030:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310031:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828797.310033:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828797.310034:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828797.310036:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828797.310037:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828797.310040:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828797.310045:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280128:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828797.310048:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828797.310050:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310054:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828797.310055:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.310056:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828797.310057:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828797.310059:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828797.310060:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280256:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828797.310062:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828797.310063:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828797.310064:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.310065:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828797.310067:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828797.310069:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828797.310071:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828797.310072:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828797.310074:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828797.310076:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280256, portal 4
00000100:00000001:3.0:1787828797.310077:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828797.310078:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:3.0:1787828797.310082:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310086:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828797.310088:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828797.310089:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828797.310091:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327280256, offset 0
00000400:00000200:3.0:1787828797.310093:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828797.310095:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828797.310097:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828797.310098:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828797.310100:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828797.310102:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828797.310103:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828797.310105:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39961:39961:256:4294967295] 192.168.203.116@tcp LPNI seq info [39961:39961:8:4294967295]
00000400:00000200:3.0:1787828797.310108:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828797.310112:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828797.310114:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.310116:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828797.310118:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828797.310120:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828797.310123:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828797.310124:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828797.310140:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.310141:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828797.310144:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828797.310146:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828797.310148:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828797.310150:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828797.310151:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828797.310154:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310159:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.310161:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310166:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.310167:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828797.310169:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828797.310171:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828797.310173:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828797.310174:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297617426
00000100:00000001:3.0:1787828797.310176:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297618092
00000100:00000001:3.0:1787828797.310178:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297617426
00000100:00000001:3.0:1787828797.310179:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297618093
00000100:00000001:3.0:1787828797.310181:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828797.310182:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.310183:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828797.310186:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828797/real 1787828797]  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828797.310190:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828797.310191:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828797.310192:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828797.310193:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828797.310194:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828797.310195:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828797.310197:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828797.310203:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310207:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828797.310208:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310209:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.310210:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828797.310212:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310216:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828797.310218:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310222:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828797.310223:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310224:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.310225:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310227:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828797.310228:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828797.310230:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828797.310232:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828797.310233:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828797.310235:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828797.310240:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280256:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828797.310242:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828797.310244:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310248:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828797.310249:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.310250:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828797.310251:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828797.310252:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828797.310254:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280384:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828797.310256:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828797.310257:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828797.310258:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828797.310260:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828797.310262:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828797.310263:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828797.310265:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828797.310268:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828797.310271:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828797.310273:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280384, portal 4
00000100:00000001:3.0:1787828797.310275:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828797.310277:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:3.0:1787828797.310282:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310289:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828797.310292:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828797.310294:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828797.310297:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327280384, offset 0
00000400:00000200:3.0:1787828797.310301:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828797.310305:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828797.310309:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828797.310312:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828797.310314:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828797.310317:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828797.310319:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828797.310322:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39962:39962:256:4294967295] 192.168.203.116@tcp LPNI seq info [39962:39962:8:4294967295]
00000400:00000200:3.0:1787828797.310325:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828797.310330:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828797.310333:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.310335:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828797.310338:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828797.310339:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787828797.310342:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828797.310344:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787828797.310346:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828797.310348:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828797.310350:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828797.310352:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828797.310354:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828797.310356:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828797.310357:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828797.310360:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310365:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.310367:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310376:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.310377:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828797.310379:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828797.310381:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828797.310383:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828797.310384:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297617431
00000100:00000001:3.0:1787828797.310386:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297618092
00000100:00000001:3.0:1787828797.310387:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297617431
00000100:00000001:3.0:1787828797.310389:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297618093
00000100:00000001:3.0:1787828797.310390:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828797.310391:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.310393:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828797.310395:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828797/real 1787828797]  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828797.310400:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828797.310401:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828797.310402:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828797.310403:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828797.310404:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828797.310405:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828797.310407:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828797.310413:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310417:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828797.310417:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310418:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.310419:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828797.310421:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310426:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828797.310429:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310433:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828797.310434:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310435:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828797.310436:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310437:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828797.310438:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828797.310440:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828797.310441:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828797.310443:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828797.310445:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828797.310450:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280384:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828797.310452:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310453:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.310455:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327280128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310459:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828797.310460:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828797.310461:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828797.310463:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828797.310464:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828797.310465:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828797.310467:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828797.310468:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:3.0:1787828797.310470:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828797.310471:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828797.310472:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:3.0:1787828797.310474:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828797.310475:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310476:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.310478:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327280256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310482:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828797.310483:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828797.310484:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828797.310485:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828797.310486:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828797.310486:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828797.310487:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828797.310488:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:3.0:1787828797.310490:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828797.310490:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828797.310491:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:3.0:1787828797.310492:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828797.310493:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310494:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828797.310496:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327280384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828797.310499:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828797.310500:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828797.310501:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828797.310502:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828797.310503:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828797.310503:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828797.310504:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828797.310505:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828797.310506:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828797.310507:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828797.310508:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828797.310509:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828797.310510:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310511:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828797.310514:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828797.310515:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828797.310516:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828797.310517:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828798.012765:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297618093, at 4297618112
00000100:00000040:2.0:1787828798.012770:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297618092, at 4297618112
00000100:00000010:3.0:1787828798.012782:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f960.
00000100:00000010:2.0:1787828798.012785:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f360.
00000020:00000001:2.0:1787828798.140872:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828798.140875:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828798.140878:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828799.164843:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828799.164848:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828799.164851:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828801.212900:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828801.212903:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828801.212905:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000008:00000020:2.0:1787828801.404789:0:16289:0:(osc_request.c:747:osc_update_next_shrink()) next time 4154 to shrink grant
00000020:00000001:2.0:1787828802.236873:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828802.236878:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828802.236881:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828802.429008:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828802.429017:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828802.429033:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828802.429052:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429092:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828802.429100:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.429111:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828802.429120:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828802.429124:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828802.429135:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327280512:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828802.429150:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828802.429165:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828802.429174:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.429187:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828802.429203:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828802.429211:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828802.429233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828802.429249:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828802.429270:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828802.429291:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280512, portal 25
00000100:00000001:3.0:1787828802.429298:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828802.429305:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:3.0:1787828802.429320:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429366:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828802.429377:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828802.429386:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828802.429395:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327280512, offset 0
00000400:00000200:3.0:1787828802.429408:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828802.429426:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.429437:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828802.429446:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828802.429458:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828802.429473:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828802.429485:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828802.429501:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39963:39963:256:4294967295] 192.168.203.116@tcp LPNI seq info [39963:39963:8:4294967295]
00000400:00000200:3.0:1787828802.429512:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.429533:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828802.429551:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.429569:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828802.429583:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828802.429594:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828802.429611:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828802.429622:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828802.429635:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.429639:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828802.429715:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828802.429723:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828802.429730:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828802.429733:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828802.429738:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828802.429742:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429753:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.429756:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429763:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.429764:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828802.429792:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828802.429798:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828802.429801:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297623213
00000100:00000010:3.0:1787828802.429807:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f960.
00000100:00000001:3.0:1787828802.429816:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828802.429823:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297623213
00000100:00000010:3.0:1787828802.429825:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fd20.
00000100:00000001:3.0:1787828802.429828:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828802.429831:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828802.429833:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.429836:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828802.429841:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802]  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828802.429850:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828802.429852:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828802.429854:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828802.429857:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828802.429858:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828802.429861:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828802.429864:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828802.429872:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429877:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828802.429879:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.429881:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.429882:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828802.429886:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429891:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828802.429895:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429903:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828802.429906:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.429907:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.429908:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.429911:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828802.429913:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828802.429916:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828802.429920:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828802.429922:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828802.429927:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828802.429958:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327280512:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828802.429967:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.429970:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.429976:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327280512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.429986:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828802.429989:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828802.429993:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828802.429997:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828802.430000:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828802.430002:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828802.430004:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828802.430007:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:3.0:1787828802.430011:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828802.430013:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828802.430015:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:3.0:1787828802.430018:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828802.430020:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430022:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430031:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828802.430033:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430035:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828802.430037:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828802.430038:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828802.430042:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430049:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828802.430050:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.430052:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828802.430054:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828802.430056:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828802.430058:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280640:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828802.430061:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828802.430064:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828802.430065:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.430067:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828802.430070:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828802.430071:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828802.430075:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828802.430078:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828802.430081:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828802.430084:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280640, portal 10
00000100:00000001:3.0:1787828802.430086:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828802.430088:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:3.0:1787828802.430094:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430100:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828802.430102:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828802.430104:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828802.430106:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327280640, offset 0
00000400:00000200:3.0:1787828802.430110:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828802.430113:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.430117:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828802.430120:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828802.430123:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828802.430125:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828802.430127:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828802.430130:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39964:39964:256:4294967295] 192.168.203.116@tcp LPNI seq info [39964:39964:8:4294967295]
00000400:00000200:3.0:1787828802.430134:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.430140:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828802.430144:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.430147:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828802.430150:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828802.430153:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828802.430157:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828802.430159:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828802.430162:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.430164:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828802.430167:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828802.430170:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828802.430173:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828802.430174:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828802.430176:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828802.430179:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430186:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.430189:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430195:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430197:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828802.430199:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828802.430203:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828802.430206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828802.430207:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297622539
00000100:00000001:3.0:1787828802.430210:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297623213
00000100:00000001:3.0:1787828802.430212:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297622539
00000100:00000001:3.0:1787828802.430214:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297623213
00000100:00000001:3.0:1787828802.430216:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828802.430218:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430219:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828802.430222:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802]  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828802.430230:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828802.430231:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828802.430233:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828802.430236:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828802.430238:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828802.430241:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828802.430245:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828802.430256:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430265:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828802.430267:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430269:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.430271:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828802.430277:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430287:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828802.430293:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430300:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828802.430302:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430304:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.430306:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430307:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828802.430310:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828802.430312:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828802.430315:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828802.430317:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828802.430320:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828802.430327:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280640:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828802.430330:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828802.430333:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430338:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828802.430339:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.430341:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828802.430343:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828802.430345:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828802.430347:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280768:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828802.430350:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828802.430352:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828802.430353:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.430355:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828802.430358:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828802.430360:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828802.430364:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828802.430366:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828802.430369:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828802.430372:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280768, portal 4
00000100:00000001:3.0:1787828802.430374:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828802.430375:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:3.0:1787828802.430380:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430386:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828802.430389:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828802.430390:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828802.430393:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327280768, offset 0
00000400:00000200:3.0:1787828802.430396:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828802.430399:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.430402:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828802.430405:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828802.430407:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828802.430409:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828802.430411:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828802.430414:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39965:39965:256:4294967295] 192.168.203.116@tcp LPNI seq info [39965:39965:8:4294967295]
00000400:00000200:3.0:1787828802.430418:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.430423:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828802.430427:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.430429:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828802.430433:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828802.430435:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828802.430439:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828802.430441:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828802.430445:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.430446:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828802.430449:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828802.430452:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828802.430454:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828802.430456:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828802.430458:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828802.430461:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430467:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.430470:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430476:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430477:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828802.430480:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828802.430482:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828802.430485:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828802.430487:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297622546
00000100:00000001:3.0:1787828802.430489:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297623213
00000100:00000001:3.0:1787828802.430491:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297622546
00000100:00000001:3.0:1787828802.430492:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297623213
00000100:00000001:3.0:1787828802.430494:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828802.430495:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430497:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828802.430500:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802]  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828802.430522:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828802.430524:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828802.430525:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828802.430527:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828802.430529:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828802.430531:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828802.430534:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828802.430542:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430548:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828802.430549:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430550:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.430552:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828802.430555:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430562:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828802.430565:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430571:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828802.430573:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430574:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.430576:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430577:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828802.430578:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828802.430580:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828802.430583:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828802.430585:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828802.430588:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828802.430594:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280768:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828802.430598:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828802.430601:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430606:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828802.430608:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.430610:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828802.430612:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828802.430614:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828802.430616:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280896:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828802.430619:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828802.430621:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828802.430623:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828802.430625:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828802.430628:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828802.430630:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828802.430632:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828802.430635:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828802.430637:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828802.430640:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327280896, portal 4
00000100:00000001:3.0:1787828802.430642:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828802.430643:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:3.0:1787828802.430649:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430655:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828802.430656:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828802.430658:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828802.430660:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327280896, offset 0
00000400:00000200:3.0:1787828802.430664:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828802.430667:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.430670:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828802.430697:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828802.430700:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828802.430703:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828802.430720:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828802.430723:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39966:39966:256:4294967295] 192.168.203.116@tcp LPNI seq info [39966:39966:8:4294967295]
00000400:00000200:3.0:1787828802.430727:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828802.430733:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828802.430737:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.430740:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828802.430743:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828802.430746:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787828802.430749:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828802.430751:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787828802.430755:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828802.430757:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828802.430760:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828802.430762:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828802.430765:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828802.430766:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828802.430785:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828802.430789:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430796:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.430799:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430805:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430806:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828802.430809:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828802.430812:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828802.430815:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828802.430817:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297622552
00000100:00000001:3.0:1787828802.430819:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297623213
00000100:00000001:3.0:1787828802.430821:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297622552
00000100:00000001:3.0:1787828802.430823:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297623213
00000100:00000001:3.0:1787828802.430824:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828802.430826:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.430827:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828802.430830:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802]  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828802.430837:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828802.430838:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828802.430840:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828802.430841:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828802.430842:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828802.430845:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828802.430848:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828802.430855:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430860:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828802.430861:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430862:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.430863:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828802.430866:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430871:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828802.430874:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430879:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828802.430881:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430882:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828802.430884:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430885:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828802.430887:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828802.430889:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828802.430892:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828802.430893:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828802.430896:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828802.430902:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327280896:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828802.430905:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430906:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.430909:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327280640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430914:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828802.430916:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828802.430918:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828802.430920:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828802.430921:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828802.430922:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828802.430924:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828802.430926:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:3.0:1787828802.430929:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828802.430931:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828802.430933:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:3.0:1787828802.430936:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828802.430938:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430941:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.430945:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327280768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430954:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828802.430955:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828802.430957:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828802.430960:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828802.430962:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828802.430964:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828802.430966:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828802.430969:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:3.0:1787828802.430972:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828802.430974:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828802.430975:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:3.0:1787828802.430979:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828802.430981:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.430983:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828802.430988:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327280896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828802.430996:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828802.430998:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828802.430999:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828802.431001:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828802.431002:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828802.431003:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828802.431005:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828802.431006:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:3.0:1787828802.431008:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828802.431009:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828802.431010:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:3.0:1787828802.431011:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828802.431012:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.431014:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828802.431017:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828802.431018:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828802.431020:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828802.431021:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828803.132759:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297623213, at 4297623232
00000100:00000010:2.0:1787828803.132785:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f960.
00000100:00000040:3.0:1787828803.132842:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297623213, at 4297623232
00000100:00000010:3.0:1787828803.132859:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd20.
00000020:00000001:2.0:1787828804.284946:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828804.284954:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828804.284970:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828805.308961:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828805.308966:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828805.308971:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828807.356842:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828807.356847:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828807.356851:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828807.548774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828807.548785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828807.548788:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828807.548791:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828807.548795:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828807.548801:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2955
00000400:00000200:2.0:1787828807.548808:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828807.548813:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828807.548816:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828807.548818:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548821:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.548822:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548825:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828807.548828:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828807.548830:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828807.548832:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828807.548835:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828807.548836:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828807.548838:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828807.548839:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548841:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.548842:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548843:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828807.548846:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828807.548848:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548850:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828807.548852:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828807.548855:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828807.548857:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828807.548858:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828807.548860:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828807.548867:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787828807.548870:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828807.548873:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828807.548874:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828807.548876:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828807.548878:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828807.548882:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828807.548888:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828807.548893:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828807.548900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828807.548908:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828807.548911:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828807.548916:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828807.548934:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828807.548937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828807.548939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828807.548940:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828807.548941:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828807.548943:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828807.548945:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2955
00000400:00000200:2.0:1787828807.548947:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828807.548949:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828807.548950:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828807.548952:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548954:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.548955:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548956:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828807.548958:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828807.548959:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828807.548960:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828807.548961:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828807.548962:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828807.548964:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828807.548965:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548966:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.548968:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548969:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828807.548970:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828807.548972:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828807.548974:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828807.548976:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828807.548981:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828807.548982:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828807.548984:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828807.548987:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828807.548990:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.548991:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787828807.548992:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.548997:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000010:2.0:1787828807.548999:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828807.549000:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1787828807.549004:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828807.549006:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828807.549008:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828807.549010:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828807.549011:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828807.549013:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828807.549015:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828807.549021:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:3.0:1787828807.549026:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828807.549028:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549031:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787828807.549032:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549033:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828807.549037:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828807.549040:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1787828807.549042:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327281024:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828807.549051:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787828807.549055:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787828807.549056:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787828807.549057:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549059:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828807.549064:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787828807.549064:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828807.549070:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:2.0:1787828807.549072:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549073:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828807.549077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787828807.549082:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828807.549083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828807.549086:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828807.549088:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:3.0:1787828807.549088:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000100:00000001:2.0:1787828807.549094:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828807.549096:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2955
00000400:00000010:3.0:1787828807.549096:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000400:00000200:2.0:1787828807.549101:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828807.549104:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000200:3.0:1787828807.549105:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281024, portal 25
00000100:00080000:2.0:1787828807.549106:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787828807.549110:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1787828807.549112:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828807.549113:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402436160 : -116696307115456 : ffff95dd86115a40)
00000100:00000001:2.0:1787828807.549116:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.549122:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:3.0:1787828807.549123:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828807.549127:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828807.549132:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828807.549134:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828807.549137:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787828807.549139:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1787828807.549140:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828807.549144:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828807.549145:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787828807.549150:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:3.0:1787828807.549150:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828807.549156:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327281024, offset 0
00000100:00000040:2.0:1787828807.549157:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.549161:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.549163:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.549168:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:3.0:1787828807.549168:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787828807.549172:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828807.549176:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828807.549177:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828807.549180:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787828807.549181:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828807.549184:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828807.549187:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549189:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828807.549190:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828807.549194:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787828807.549197:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00000001:2.0:1787828807.549198:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828807.549201:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549203:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000010:2.0:1787828807.549206:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000400:00000200:3.0:1787828807.549208:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:2.0:1787828807.549211:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:3.0:1787828807.549213:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:2.0:1787828807.549216:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828807.549219:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549220:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39967:39967:256:4294967295] 192.168.203.116@tcp LPNI seq info [39967:39967:8:4294967295]
00000100:00000001:2.0:1787828807.549223:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828807.549227:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787828807.549229:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787828807.549230:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828807.549235:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828807.549240:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549244:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00100000:2.0:1787828807.549249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1787828807.549255:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828807.549261:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000010:3.0:1787828807.549263:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000100:00000001:2.0:1787828807.549264:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787828807.549271:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787828807.549276:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1787828807.549277:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828807.549288:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828807.549290:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787828807.549293:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00080000:2.0:1787828807.549294:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828807.549300:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000100:3.0:1787828807.549301:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828807.549303:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000010:3.0:1787828807.549305:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000100:00080000:2.0:1787828807.549306:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828807.549310:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828807.549312:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2955
00000400:00000200:3.0:1787828807.549313:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787828807.549318:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828807.549323:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:3.0:1787828807.549324:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1787828807.549325:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828807.549332:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828807.549332:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828807.549336:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:2.0:1787828807.549337:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828807.549342:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828807.549343:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787828807.549344:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828807.549348:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828807.549350:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000200:3.0:1787828807.549351:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787828807.549352:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828807.549355:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828807.549360:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828807.549364:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828807.549369:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828807.549369:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787828807.549373:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828807.549376:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828807.549378:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828807.549383:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828807.549389:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:3.0:1787828807.549390:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549394:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:2.0:1787828807.549397:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828807.549399:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828807.549401:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549405:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828807.549409:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297628332
00000100:00000010:3.0:1787828807.549413:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fd20.
02000000:00000001:2.0:1787828807.549415:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828807.549418:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:2.0:1787828807.549421:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828807.549425:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549425:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297628332
00000100:00000010:3.0:1787828807.549431:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f120.
00000100:00000001:2.0:1787828807.549432:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828807.549436:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000008:00100000:2.0:1787828807.549438:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828807.549440:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1787828807.549442:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549444:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828807.549445:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549449:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:2.0:1787828807.549452:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000100:00100000:3.0:1787828807.549454:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807]  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787828807.549458:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828807.549466:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:3.0:1787828807.549467:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:2.0:1787828807.549470:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787828807.549470:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828807.549473:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000100:00000001:2.0:1787828807.549476:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787828807.549477:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
02000000:00000001:2.0:1787828807.549481:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787828807.549481:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828807.549483:0:2346:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:2.0:1787828807.549486:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828807.549487:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787828807.549497:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000200:3.0:1787828807.549498:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828807.549504:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549505:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828807.549509:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549511:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:2.0:1787828807.549514:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828807.549514:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828807.549523:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828807.549527:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828807.549531:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549533:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828807.549537:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787828807.549540:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828807.549565:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549566:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828807.549571:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828807.549572:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2965)
00000100:00000001:3.0:1787828807.549572:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.549575:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549578:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828807.549580:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828807.549582:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828807.549585:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828807.549587:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828807.549591:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828807.549596:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327281024:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828807.549599:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549601:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.549603:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115a40 x1874671327281024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549608:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828807.549609:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828807.549612:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828807.549614:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828807.549616:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828807.549617:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828807.549619:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828807.549620:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828807.549623:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828807.549625:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828807.549626:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115a40.
00000100:00000001:3.0:1787828807.549628:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828807.549629:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549630:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549636:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828807.549637:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549639:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828807.549640:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828807.549641:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828807.549643:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549648:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828807.549649:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549650:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828807.549651:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828807.549653:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828807.549684:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281152:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828807.549687:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828807.549688:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828807.549689:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549690:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828807.549703:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828807.549705:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549707:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828807.549709:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828807.549711:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828807.549713:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281152, portal 10
00000100:00000001:3.0:1787828807.549715:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828807.549716:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:3.0:1787828807.549721:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549725:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828807.549727:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828807.549728:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828807.549730:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327281152, offset 0
00000400:00000200:3.0:1787828807.549749:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828807.549752:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828807.549755:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828807.549757:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828807.549759:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828807.549761:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828807.549762:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828807.549764:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39968:39968:256:4294967295] 192.168.203.116@tcp LPNI seq info [39968:39968:8:4294967295]
00000400:00000200:3.0:1787828807.549767:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828807.549772:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828807.549774:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828807.549777:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828807.549780:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828807.549782:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828807.549785:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828807.549786:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787828807.549789:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828807.549790:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828807.549792:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828807.549795:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828807.549797:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828807.549798:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828807.549800:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828807.549802:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549812:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.549814:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549818:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549819:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828807.549822:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828807.549824:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828807.549826:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828807.549828:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297627659
00000100:00000001:3.0:1787828807.549830:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297628332
00000100:00000001:3.0:1787828807.549831:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297627659
00000100:00000001:3.0:1787828807.549833:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297628332
00000100:00000001:3.0:1787828807.549834:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828807.549835:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.549837:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828807.549839:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807]  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828807.549844:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828807.549845:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828807.549846:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828807.549847:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828807.549848:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828807.549850:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828807.549852:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828807.549857:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549861:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828807.549863:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549864:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.549865:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828807.549867:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549871:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828807.549874:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549878:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828807.549879:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549880:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.549881:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.549882:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828807.549884:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828807.549885:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828807.549887:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828807.549888:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828807.549891:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828807.549895:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281152:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828807.549898:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828807.549900:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549904:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828807.549905:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549906:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828807.549908:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828807.549909:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828807.549911:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281280:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828807.549913:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828807.549914:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828807.549916:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.549917:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828807.549919:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828807.549920:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.549922:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828807.549924:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828807.549926:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828807.549928:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281280, portal 4
00000100:00000001:3.0:1787828807.549929:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828807.549930:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:3.0:1787828807.549934:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549938:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828807.549940:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828807.549941:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828807.549942:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327281280, offset 0
00000400:00000200:3.0:1787828807.549945:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828807.549947:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828807.549949:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828807.549951:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828807.549952:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828807.549954:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828807.549956:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828807.549957:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39969:39969:256:4294967295] 192.168.203.116@tcp LPNI seq info [39969:39969:8:4294967295]
00000400:00000200:3.0:1787828807.549960:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828807.549964:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828807.549967:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828807.549968:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828807.549971:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828807.549973:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828807.549975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828807.549977:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787828807.549980:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828807.549981:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828807.549983:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828807.549985:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828807.549987:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828807.549988:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828807.549989:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828807.549991:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.549996:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.549998:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550003:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.550004:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828807.550005:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828807.550007:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828807.550009:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828807.550010:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297627666
00000100:00000001:3.0:1787828807.550012:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297628332
00000100:00000001:3.0:1787828807.550013:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297627666
00000100:00000001:3.0:1787828807.550015:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297628332
00000100:00000001:3.0:1787828807.550016:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828807.550017:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.550018:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828807.550020:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807]  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828807.550025:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828807.550026:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828807.550027:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828807.550028:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828807.550029:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828807.550031:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828807.550033:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828807.550038:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550041:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828807.550042:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550043:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.550044:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828807.550046:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550050:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828807.550052:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550056:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828807.550057:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550059:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.550059:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550061:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828807.550062:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828807.550064:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828807.550066:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828807.550067:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828807.550069:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828807.550078:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281280:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828807.550081:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828807.550083:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550087:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828807.550087:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.550089:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828807.550090:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828807.550091:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828807.550092:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281408:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828807.550094:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828807.550095:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828807.550096:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828807.550097:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828807.550099:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828807.550100:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828807.550102:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828807.550103:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828807.550105:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828807.550107:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281408, portal 4
00000100:00000001:3.0:1787828807.550108:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828807.550109:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:3.0:1787828807.550112:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550116:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828807.550118:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828807.550119:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828807.550120:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327281408, offset 0
00000400:00000200:3.0:1787828807.550122:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828807.550124:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828807.550126:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828807.550128:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828807.550130:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828807.550132:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828807.550133:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828807.550135:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39970:39970:256:4294967295] 192.168.203.116@tcp LPNI seq info [39970:39970:8:4294967295]
00000400:00000200:3.0:1787828807.550138:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828807.550142:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828807.550144:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828807.550146:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828807.550148:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828807.550149:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787828807.550152:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828807.550154:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787828807.550156:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828807.550157:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828807.550159:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828807.550161:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828807.550163:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828807.550164:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828807.550165:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828807.550167:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550172:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.550174:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550179:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.550180:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828807.550181:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828807.550183:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828807.550185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828807.550186:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297627671
00000100:00000001:3.0:1787828807.550188:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297628332
00000100:00000001:3.0:1787828807.550190:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297627671
00000100:00000001:3.0:1787828807.550191:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297628332
00000100:00000001:3.0:1787828807.550192:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828807.550193:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.550194:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828807.550196:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807]  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828807.550201:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828807.550201:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828807.550202:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828807.550203:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828807.550205:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828807.550206:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828807.550208:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828807.550213:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550217:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828807.550218:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550219:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.550219:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828807.550221:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550225:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828807.550228:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550231:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828807.550233:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550234:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828807.550235:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550237:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828807.550238:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828807.550239:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828807.550241:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828807.550242:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828807.550245:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828807.550249:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281408:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828807.550251:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550252:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.550254:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327281152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550258:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828807.550259:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828807.550260:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828807.550262:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828807.550262:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828807.550263:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828807.550265:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828807.550266:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:3.0:1787828807.550267:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828807.550268:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828807.550269:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828807.550271:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828807.550272:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550273:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.550275:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327281280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550284:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828807.550285:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828807.550286:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828807.550287:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828807.550288:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828807.550289:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828807.550290:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828807.550291:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:3.0:1787828807.550292:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828807.550293:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828807.550293:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:3.0:1787828807.550295:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828807.550296:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550297:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828807.550298:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327281408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828807.550302:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828807.550303:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828807.550303:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828807.550305:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828807.550305:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828807.550306:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828807.550307:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828807.550308:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828807.550309:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828807.550310:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828807.550310:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:3.0:1787828807.550312:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828807.550312:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550314:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828807.550316:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828807.550316:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828807.550318:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828807.550319:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828808.252755:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297628332, at 4297628352
00000100:00000040:3.0:1787828808.252755:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297628332, at 4297628352
00000100:00000010:2.0:1787828808.252772:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd20.
00000100:00000010:3.0:1787828808.252774:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f120.
00000020:00000001:2.0:1787828808.380936:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828808.380941:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828808.380946:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828810.428954:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828810.428960:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828810.428965:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828811.452866:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828811.452871:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828811.452875:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828812.669286:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828812.669295:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828812.669303:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828812.669320:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.669351:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828812.669356:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.669364:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828812.669372:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828812.669378:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828812.669390:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327281536:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828812.669401:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828812.669418:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828812.669424:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.669431:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828812.669441:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828812.669448:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828812.669468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828812.669476:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828812.669493:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828812.669515:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281536, portal 25
00000100:00000001:3.0:1787828812.669521:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828812.669528:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:3.0:1787828812.669549:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.669582:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828812.669597:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828812.669607:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828812.669617:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327281536, offset 0
00000400:00000200:3.0:1787828812.669649:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828812.669769:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.669786:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828812.669807:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828812.669819:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828812.669830:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828812.669842:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828812.669856:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39971:39971:256:4294967295] 192.168.203.116@tcp LPNI seq info [39971:39971:8:4294967295]
00000400:00000200:3.0:1787828812.669876:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.669904:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828812.669919:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.669931:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828812.669951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828812.669960:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828812.669988:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828812.669999:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828812.670014:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.670020:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828812.670037:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828812.670063:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828812.670076:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828812.670085:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828812.670100:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828812.670116:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670156:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.670170:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670200:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.670209:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828812.670218:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828812.670232:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828812.670241:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297633453
00000100:00000010:3.0:1787828812.670265:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f120.
00000100:00000001:3.0:1787828812.670277:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828812.670291:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297633453
00000100:00000010:3.0:1787828812.670301:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f540.
00000100:00000001:3.0:1787828812.670310:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828812.670323:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828812.670331:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.670341:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828812.670354:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812]  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828812.670379:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828812.670386:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828812.670392:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828812.670410:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828812.670414:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828812.670423:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828812.670438:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828812.670487:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670498:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828812.670501:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.670504:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.670507:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828812.670513:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670522:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828812.670528:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670546:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828812.670549:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.670551:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.670554:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.670557:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828812.670560:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828812.670564:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828812.670568:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828812.670571:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828812.670577:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828812.670587:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327281536:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828812.670592:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.670595:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.670598:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327281536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670611:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828812.670615:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828812.670619:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828812.670622:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828812.670624:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828812.670627:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828812.670630:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828812.670633:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:3.0:1787828812.670637:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828812.670639:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828812.670642:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:3.0:1787828812.670645:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828812.670647:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.670650:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.670685:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828812.670688:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.670691:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828812.670693:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828812.670695:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828812.670721:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670731:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828812.670733:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.670736:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828812.670738:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828812.670741:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828812.670745:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281664:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828812.670755:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828812.670757:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828812.670760:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.670762:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828812.670766:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828812.670769:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828812.670773:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828812.670777:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828812.670781:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828812.670785:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281664, portal 10
00000100:00000001:3.0:1787828812.670788:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828812.670790:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:3.0:1787828812.670798:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670806:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828812.670810:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828812.670812:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828812.670815:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327281664, offset 0
00000400:00000200:3.0:1787828812.670820:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828812.670825:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.670830:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828812.670834:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828812.670837:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828812.670840:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828812.670844:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828812.670848:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39972:39972:256:4294967295] 192.168.203.116@tcp LPNI seq info [39972:39972:8:4294967295]
00000400:00000200:3.0:1787828812.670854:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.670871:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828812.670877:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.670881:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828812.670886:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828812.670890:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828812.670896:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828812.670899:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828812.670904:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.670906:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828812.670911:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828812.670915:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828812.670919:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828812.670921:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828812.670925:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828812.670929:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670939:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.670943:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.670952:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.670954:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828812.670958:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828812.670962:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828812.670977:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828812.670980:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297632780
00000100:00000001:3.0:1787828812.670983:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297633453
00000100:00000001:3.0:1787828812.670986:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297632780
00000100:00000001:3.0:1787828812.670989:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297633453
00000100:00000001:3.0:1787828812.670992:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828812.670995:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.670997:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828812.671002:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812]  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828812.671012:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828812.671014:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828812.671016:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828812.671018:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828812.671020:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828812.671023:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828812.671026:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828812.671037:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671045:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828812.671046:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671049:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.671050:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828812.671054:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671063:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828812.671067:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671075:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828812.671078:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671080:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.671082:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671084:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828812.671086:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828812.671089:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828812.671094:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828812.671097:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828812.671102:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828812.671112:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281664:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828812.671116:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828812.671120:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671128:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828812.671129:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.671132:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828812.671134:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828812.671136:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828812.671140:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281792:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828812.671143:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828812.671145:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828812.671147:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.671149:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828812.671153:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828812.671155:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828812.671159:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828812.671162:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828812.671165:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828812.671169:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281792, portal 4
00000100:00000001:3.0:1787828812.671171:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828812.671173:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:3.0:1787828812.671180:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671188:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828812.671192:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828812.671194:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828812.671197:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327281792, offset 0
00000400:00000200:3.0:1787828812.671202:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828812.671215:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.671219:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828812.671222:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828812.671226:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828812.671228:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828812.671231:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828812.671235:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39973:39973:256:4294967295] 192.168.203.116@tcp LPNI seq info [39973:39973:8:4294967295]
00000400:00000200:3.0:1787828812.671240:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.671248:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828812.671253:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.671257:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828812.671261:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828812.671264:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828812.671269:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828812.671272:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828812.671308:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.671311:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828812.671315:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828812.671320:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828812.671324:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828812.671326:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828812.671330:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828812.671335:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671345:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.671348:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671357:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.671359:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828812.671364:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828812.671368:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828812.671372:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828812.671375:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297632787
00000100:00000001:3.0:1787828812.671378:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297633453
00000100:00000001:3.0:1787828812.671381:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297632787
00000100:00000001:3.0:1787828812.671384:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297633453
00000100:00000001:3.0:1787828812.671387:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828812.671390:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.671392:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828812.671397:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812]  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828812.671414:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828812.671415:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828812.671417:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828812.671420:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828812.671422:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828812.671424:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828812.671427:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828812.671438:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671446:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828812.671448:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671450:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.671452:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828812.671456:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671464:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828812.671468:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671488:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828812.671491:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671493:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.671495:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671497:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828812.671499:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828812.671502:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828812.671506:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828812.671508:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828812.671513:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828812.671522:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281792:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828812.671526:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828812.671530:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671538:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828812.671539:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.671542:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828812.671544:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828812.671547:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828812.671550:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281920:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828812.671554:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828812.671556:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828812.671558:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828812.671560:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828812.671564:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828812.671567:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828812.671570:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828812.671574:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828812.671577:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828812.671580:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327281920, portal 4
00000100:00000001:3.0:1787828812.671583:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828812.671585:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:3.0:1787828812.671591:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671598:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828812.671601:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828812.671604:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828812.671607:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327281920, offset 0
00000400:00000200:3.0:1787828812.671611:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828812.671615:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.671620:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828812.671624:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828812.671637:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828812.671640:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828812.671643:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828812.671647:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39974:39974:256:4294967295] 192.168.203.116@tcp LPNI seq info [39974:39974:8:4294967295]
00000400:00000200:3.0:1787828812.671652:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828812.671688:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828812.671694:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.671720:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828812.671726:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828812.671729:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787828812.671744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828812.671747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787828812.671752:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828812.671755:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828812.671759:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828812.671763:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828812.671767:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828812.671769:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828812.671773:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828812.671777:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671787:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.671791:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671810:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.671812:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828812.671816:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828812.671820:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828812.671824:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828812.671826:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297632793
00000100:00000001:3.0:1787828812.671829:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297633453
00000100:00000001:3.0:1787828812.671832:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297632793
00000100:00000001:3.0:1787828812.671835:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297633453
00000100:00000001:3.0:1787828812.671837:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828812.671840:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.671842:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828812.671847:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812]  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828812.671856:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828812.671858:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828812.671860:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828812.671862:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828812.671873:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828812.671875:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828812.671879:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828812.671889:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671905:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828812.671907:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671909:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.671911:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828812.671915:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671923:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828812.671927:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671935:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828812.671938:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671940:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828812.671941:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671943:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828812.671946:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828812.671949:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828812.671952:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828812.671955:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828812.671959:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828812.671968:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327281920:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828812.671973:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.671975:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.671979:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327281664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.671987:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828812.671989:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828812.671993:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828812.671996:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828812.671998:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828812.672000:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828812.672003:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828812.672005:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:3.0:1787828812.672008:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828812.672010:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828812.672012:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:3.0:1787828812.672015:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828812.672017:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.672019:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.672022:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327281792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.672031:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828812.672032:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828812.672034:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828812.672037:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828812.672039:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828812.672040:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828812.672042:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828812.672044:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:3.0:1787828812.672046:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828812.672048:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828812.672049:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:3.0:1787828812.672052:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828812.672053:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.672055:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828812.672059:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327281920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828812.672066:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828812.672068:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828812.672069:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828812.672072:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828812.672073:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828812.672074:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828812.672077:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828812.672079:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:3.0:1787828812.672082:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828812.672083:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828812.672085:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:3.0:1787828812.672088:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828812.672089:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.672091:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828812.672096:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828812.672098:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828812.672100:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828812.672102:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828813.372881:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297633453, at 4297633472
00000100:00000040:2.0:1787828813.372893:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297633453, at 4297633472
00000100:00000010:3.0:1787828813.372902:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f540.
00000100:00000010:2.0:1787828813.372912:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f120.
00000020:00000001:2.0:1787828813.500918:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828813.500923:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828813.500926:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828814.524917:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828814.524924:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828814.524930:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828816.572888:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828816.572892:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828816.572895:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828817.596880:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828817.596884:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828817.596888:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828817.789035:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828817.789041:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828817.789045:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828817.789054:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789076:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828817.789080:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.789086:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828817.789091:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828817.789096:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828817.789105:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327282048:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828817.789112:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828817.789117:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828817.789120:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.789125:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828817.789132:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828817.789138:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828817.789148:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828817.789171:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828817.789181:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828817.789193:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282048, portal 25
00000100:00000001:3.0:1787828817.789201:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828817.789205:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:3.0:1787828817.789219:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789241:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828817.789247:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828817.789251:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828817.789257:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327282048, offset 0
00000400:00000200:3.0:1787828817.789268:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828817.789282:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.789291:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828817.789298:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828817.789308:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828817.789313:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828817.789320:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828817.789330:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39975:39975:256:4294967295] 192.168.203.116@tcp LPNI seq info [39975:39975:8:4294967295]
00000400:00000200:3.0:1787828817.789342:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.789362:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828817.789375:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.789383:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828817.789395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828817.789402:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828817.789408:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828817.789411:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787828817.789434:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.789438:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828817.789446:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828817.789454:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828817.789459:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828817.789463:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828817.789470:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828817.789476:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789492:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.789498:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789532:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.789540:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828817.789549:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828817.789559:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828817.789568:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297638572
00000100:00000010:3.0:1787828817.789578:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f540.
00000100:00000001:3.0:1787828817.789589:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828817.789608:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297638572
00000100:00000010:3.0:1787828817.789618:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f1e0.
00000100:00000001:3.0:1787828817.789627:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828817.789632:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828817.789640:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.789643:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828817.789652:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817]  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828817.789723:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828817.789726:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828817.789729:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828817.789733:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828817.789737:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828817.789742:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828817.789748:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828817.789760:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789770:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828817.789773:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.789775:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.789776:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828817.789779:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789784:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828817.789788:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789794:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828817.789796:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.789797:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.789798:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.789800:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828817.789801:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828817.789803:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828817.789805:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828817.789807:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828817.789809:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828817.789814:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327282048:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828817.789817:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.789819:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.789821:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327282048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789825:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828817.789827:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828817.789828:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828817.789830:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828817.789832:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828817.789833:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828817.789835:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828817.789836:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:3.0:1787828817.789839:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828817.789840:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828817.789842:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:3.0:1787828817.789844:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828817.789845:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.789846:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.789851:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828817.789852:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.789854:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828817.789855:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828817.789856:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828817.789858:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789863:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828817.789864:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.789865:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828817.789867:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828817.789868:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828817.789870:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282176:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828817.789872:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828817.789873:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828817.789874:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.789875:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828817.789878:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828817.789879:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828817.789882:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828817.789883:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828817.789885:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828817.789888:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282176, portal 10
00000100:00000001:3.0:1787828817.789889:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828817.789890:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:3.0:1787828817.789894:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789899:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828817.789901:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828817.789902:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828817.789903:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327282176, offset 0
00000400:00000200:3.0:1787828817.789906:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828817.789909:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.789911:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828817.789913:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828817.789915:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828817.789916:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828817.789918:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828817.789920:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39976:39976:256:4294967295] 192.168.203.116@tcp LPNI seq info [39976:39976:8:4294967295]
00000400:00000200:3.0:1787828817.789923:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.789928:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828817.789931:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.789933:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828817.789935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828817.789937:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828817.789940:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828817.789942:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787828817.789945:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.789946:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828817.789949:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828817.789951:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828817.789953:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828817.789954:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828817.789956:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828817.789958:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789963:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.789965:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.789974:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.789975:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828817.789977:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828817.789980:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828817.789982:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828817.789984:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297637899
00000100:00000001:3.0:1787828817.789986:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297638572
00000100:00000001:3.0:1787828817.789987:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297637899
00000100:00000001:3.0:1787828817.789989:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297638572
00000100:00000001:3.0:1787828817.789990:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828817.789992:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.789993:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828817.789995:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817]  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828817.789999:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828817.790000:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828817.790001:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828817.790003:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828817.790004:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828817.790005:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828817.790007:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828817.790012:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790016:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828817.790017:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790018:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.790018:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828817.790021:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790025:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828817.790028:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790032:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828817.790033:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790034:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.790035:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790036:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828817.790037:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828817.790039:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828817.790041:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828817.790042:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828817.790044:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828817.790049:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282176:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828817.790051:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828817.790053:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790057:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828817.790058:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.790059:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828817.790061:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828817.790062:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828817.790064:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282304:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828817.790066:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828817.790067:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828817.790068:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.790069:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828817.790071:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828817.790072:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828817.790074:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828817.790076:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828817.790078:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828817.790080:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282304, portal 4
00000100:00000001:3.0:1787828817.790081:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828817.790082:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:3.0:1787828817.790086:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790090:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828817.790092:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828817.790093:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828817.790095:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327282304, offset 0
00000400:00000200:3.0:1787828817.790097:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828817.790099:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.790101:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828817.790102:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828817.790104:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828817.790105:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828817.790107:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828817.790109:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39977:39977:256:4294967295] 192.168.203.116@tcp LPNI seq info [39977:39977:8:4294967295]
00000400:00000200:3.0:1787828817.790111:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.790115:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828817.790118:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.790120:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828817.790122:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828817.790123:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828817.790126:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828817.790127:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787828817.790130:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.790132:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828817.790134:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828817.790136:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828817.790138:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828817.790139:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828817.790140:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828817.790142:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790147:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.790149:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790153:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.790154:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828817.790156:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828817.790157:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828817.790159:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828817.790160:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297637906
00000100:00000001:3.0:1787828817.790162:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297638572
00000100:00000001:3.0:1787828817.790164:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297637906
00000100:00000001:3.0:1787828817.790165:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297638572
00000100:00000001:3.0:1787828817.790166:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828817.790167:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.790168:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828817.790171:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817]  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828817.790175:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828817.790175:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828817.790176:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828817.790178:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828817.790179:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828817.790197:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828817.790200:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828817.790206:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790211:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828817.790212:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790213:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.790214:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828817.790217:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790222:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828817.790225:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790230:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828817.790232:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790233:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.790234:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790235:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828817.790236:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828817.790238:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828817.790241:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828817.790242:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828817.790245:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828817.790251:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282304:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828817.790254:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828817.790257:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790263:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828817.790264:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.790266:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828817.790268:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828817.790269:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828817.790271:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282432:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828817.790273:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828817.790275:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828817.790276:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828817.790277:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828817.790280:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828817.790281:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828817.790283:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828817.790285:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828817.790287:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828817.790289:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282432, portal 4
00000100:00000001:3.0:1787828817.790291:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828817.790292:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:3.0:1787828817.790296:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790302:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828817.790303:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828817.790305:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828817.790307:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327282432, offset 0
00000400:00000200:3.0:1787828817.790309:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828817.790312:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.790314:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828817.790316:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828817.790318:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828817.790320:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828817.790322:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828817.790325:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39978:39978:256:4294967295] 192.168.203.116@tcp LPNI seq info [39978:39978:8:4294967295]
00000400:00000200:3.0:1787828817.790328:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828817.790333:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828817.790337:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.790339:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828817.790342:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828817.790343:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787828817.790347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828817.790349:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787828817.790352:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828817.790354:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288ff00.
00000400:00000200:3.0:1787828817.790356:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828817.790359:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828817.790361:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99880
00000400:00000010:3.0:1787828817.790362:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99880.
00000100:00000001:3.0:1787828817.790364:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828817.790366:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790372:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.790375:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790379:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.790381:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828817.790383:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828817.790385:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828817.790388:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828817.790389:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297637911
00000100:00000001:3.0:1787828817.790391:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297638572
00000100:00000001:3.0:1787828817.790393:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297637911
00000100:00000001:3.0:1787828817.790395:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297638572
00000100:00000001:3.0:1787828817.790396:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828817.790398:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.790399:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828817.790402:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817]  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828817.790408:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828817.790409:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828817.790410:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828817.790412:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828817.790413:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828817.790414:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828817.790417:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828817.790423:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790427:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828817.790429:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790446:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.790448:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828817.790450:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790454:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828817.790457:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790461:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828817.790462:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790463:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828817.790464:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790465:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828817.790466:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828817.790467:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828817.790470:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828817.790471:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828817.790473:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828817.790478:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282432:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828817.790480:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790481:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.790483:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327282176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790487:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828817.790488:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828817.790489:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828817.790490:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828817.790491:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828817.790492:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828817.790493:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828817.790494:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4800.
02000000:00000001:3.0:1787828817.790496:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828817.790497:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828817.790498:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:3.0:1787828817.790499:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828817.790500:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790501:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.790503:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327282304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790507:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828817.790508:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828817.790509:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828817.790510:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828817.790511:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828817.790511:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828817.790512:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828817.790514:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:3.0:1787828817.790532:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828817.790533:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828817.790534:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:3.0:1787828817.790537:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828817.790538:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790539:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828817.790541:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327282432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828817.790546:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828817.790547:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828817.790548:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828817.790550:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828817.790551:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828817.790551:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828817.790553:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828817.790554:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:3.0:1787828817.790555:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828817.790556:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828817.790557:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:3.0:1787828817.790559:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828817.790560:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790561:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828817.790563:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828817.790565:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828817.790566:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828817.790567:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828818.492785:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297638572, at 4297638592
00000100:00000040:3.0:1787828818.492785:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297638572, at 4297638592
00000100:00000010:3.0:1787828818.492804:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f1e0.
00000100:00000010:2.0:1787828818.492812:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f540.
00000020:00000001:2.0:1787828819.644911:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828819.644914:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828819.644918:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828820.668950:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828820.668956:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828820.668960:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828822.396820:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828822.396828:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828822.396831:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828822.396837:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828822.396842:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828822.396846:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828822.396849:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828822.396852:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828822.396856:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828822.716833:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828822.716837:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828822.716841:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828822.909004:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828822.909010:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828822.909014:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828822.909023:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909042:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828822.909044:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909048:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828822.909052:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828822.909054:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828822.909059:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327282560:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828822.909065:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828822.909070:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828822.909074:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909078:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828822.909083:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828822.909088:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828822.909098:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828822.909102:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828822.909109:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828822.909116:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282560, portal 25
00000100:00000001:3.0:1787828822.909121:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828822.909124:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:3.0:1787828822.909134:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909150:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828822.909158:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828822.909164:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99880.
00000100:00000200:3.0:1787828822.909171:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327282560, offset 0
00000400:00000200:3.0:1787828822.909182:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828822.909195:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.909205:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828822.909213:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828822.909221:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828822.909227:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828822.909242:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828822.909254:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39979:39979:256:4294967295] 192.168.203.116@tcp LPNI seq info [39979:39979:8:4294967295]
00000400:00000200:3.0:1787828822.909263:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.909278:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828822.909289:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828822.909296:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288ff00.
00000800:00000200:3.0:1787828822.909305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828822.909311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828822.909334:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828822.909338:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828822.909341:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297643692
00000100:00000010:3.0:1787828822.909348:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f1e0.
00000100:00000001:3.0:1787828822.909356:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828822.909364:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297643692
00000100:00000010:3.0:1787828822.909368:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f780.
00000100:00000001:3.0:1787828822.909371:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828822.909376:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828822.909381:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.909384:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.909389:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828822.909397:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828822.909401:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828822.909408:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828822.909413:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828822.909416:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828822.909425:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909437:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828822.909440:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909443:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828822.909447:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828822.909450:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828822.909456:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282688:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828822.909462:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828822.909465:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828822.909469:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909474:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828822.909479:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828822.909483:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828822.909489:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828822.909493:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98dc0.
00000400:00000010:3.0:1787828822.909500:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828822.909507:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282688, portal 10
00000100:00000001:3.0:1787828822.909510:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828822.909513:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:3.0:1787828822.909522:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909535:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828822.909539:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828822.909545:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99f68.
00000100:00000200:3.0:1787828822.909548:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327282688, offset 0
00000400:00000200:3.0:1787828822.909556:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828822.909565:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.909572:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828822.909581:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828822.909585:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828822.909590:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828822.909595:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828822.909601:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39980:39980:255:4294967295] 192.168.203.116@tcp LPNI seq info [39980:39980:7:4294967295]
00000400:00000200:3.0:1787828822.909611:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.909624:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828822.909645:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828822.909649:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f800.
00000800:00000200:3.0:1787828822.909690:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828822.909695:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828822.909697:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828822.909700:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828822.909702:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828822.909704:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297643019
00000100:00000001:3.0:1787828822.909725:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297643692
00000100:00000001:3.0:1787828822.909728:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297643019
00000100:00000001:3.0:1787828822.909730:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297643692
00000100:00000001:3.0:1787828822.909732:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828822.909734:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.909737:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828822.909741:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909749:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828822.909751:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909753:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828822.909755:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828822.909757:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828822.909760:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282816:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828822.909763:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828822.909765:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828822.909767:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909768:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828822.909772:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828822.909774:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828822.909778:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828822.909781:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828822.909785:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828822.909788:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282816, portal 4
00000100:00000001:3.0:1787828822.909801:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828822.909802:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:3.0:1787828822.909808:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909814:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828822.909817:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828822.909819:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99088.
00000100:00000200:3.0:1787828822.909822:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327282816, offset 0
00000400:00000200:3.0:1787828822.909827:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828822.909832:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.909835:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828822.909838:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828822.909840:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828822.909842:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828822.909845:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828822.909848:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39981:39981:254:4294967295] 192.168.203.116@tcp LPNI seq info [39981:39981:6:4294967295]
00000400:00000200:3.0:1787828822.909851:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.909857:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828822.909862:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828822.909867:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f100.
00000800:00000200:3.0:1787828822.909871:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828822.909875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828822.909878:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828822.909881:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828822.909885:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828822.909886:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297643026
00000100:00000001:3.0:1787828822.909898:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297643692
00000100:00000001:3.0:1787828822.909901:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297643026
00000100:00000001:3.0:1787828822.909903:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297643692
00000100:00000001:3.0:1787828822.909905:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828822.909907:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.909910:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.909912:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828822.909915:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828822.909917:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828822.909919:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828822.909921:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828822.909923:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828822.909927:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909945:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828822.909947:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909950:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828822.909951:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828822.909954:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828822.909957:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282944:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828822.909960:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828822.909962:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828822.909963:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828822.909965:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828822.909969:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828822.909971:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828822.909974:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828822.909977:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828822.909981:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828822.909984:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327282944, portal 4
00000100:00000001:3.0:1787828822.909986:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828822.909987:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:3.0:1787828822.909992:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828822.909998:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828822.910000:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828822.910003:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99110.
00000100:00000200:3.0:1787828822.910005:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327282944, offset 0
00000400:00000200:3.0:1787828822.910009:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828822.910013:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.910016:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828822.910019:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828822.910021:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828822.910023:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828822.910025:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828822.910029:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39982:39982:253:4294967295] 192.168.203.116@tcp LPNI seq info [39982:39982:5:4294967295]
00000400:00000200:3.0:1787828822.910032:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828822.910038:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828822.910042:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828822.910045:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f200.
00000800:00000200:3.0:1787828822.910049:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828822.910052:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828822.910055:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828822.910057:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828822.910060:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828822.910062:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297643031
00000100:00000001:3.0:1787828822.910065:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297643692
00000100:00000001:3.0:1787828822.910067:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297643031
00000100:00000001:3.0:1787828822.910069:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297643692
00000100:00000001:3.0:1787828822.910072:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828822.910073:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.910076:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.910078:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828822.910081:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828822.910082:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828822.910084:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828822.910086:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828822.910087:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828822.910089:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828823.420757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828823.420764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828823.420766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828823.420772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828823.420775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828823.420779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828823.420781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828823.420783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828823.420786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828823.612734:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297643692, at 4297643712
00000100:00000010:3.0:1787828823.612747:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f780.
00000100:00000040:2.0:1787828823.612771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297643692, at 4297643712
00000100:00000010:2.0:1787828823.612783:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f1e0.
00000020:00000001:2.0:1787828823.740870:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828823.740875:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828823.740879:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828824.444789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828824.444796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828824.444798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828824.444804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828824.444808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828824.444812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828824.444815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828824.444817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828824.444820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828825.468789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828825.468797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828825.468800:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828825.468806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828825.468809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828825.468813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828825.468816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828825.468819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828825.468821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828825.788838:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828825.788843:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828825.788847:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828825.981390:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828825.981397:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828825.981402:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828825.981415:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828825]  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828825.981444:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828825.981449:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828825.981453:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98dc0.
00000400:00000200:3.0:1787828825.981458:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99a18
00000400:00000010:3.0:1787828825.981461:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99a18.
00000100:00000001:3.0:1787828825.981465:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828825.981475:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828825.981519:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.981540:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828825.981544:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.981550:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.981555:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828825.981565:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.981585:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828825.981617:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.981648:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828825.981711:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.981715:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.981718:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.981721:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828825.981741:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828825.981997:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828825.982005:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828825.982008:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828825.982014:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828825.982032:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282688:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828825.982038:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828825.982042:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828825]  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828825.982052:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828825.982054:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828825.982056:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828825.982059:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828825.982061:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828825.982064:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828825.982068:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828825.982078:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982085:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828825.982087:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982089:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.982090:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828825.982094:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982101:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828825.982105:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982114:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828825.982117:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982118:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.982120:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982128:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828825.982130:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828825.982132:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828825.982137:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828825.982139:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828825.982144:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828825.982152:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282816:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828825.982157:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828825.982161:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828825]  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828825.982170:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828825.982171:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828825.982174:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828825.982176:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828825.982178:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828825.982181:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828825.982184:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828825.982192:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982199:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828825.982200:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982202:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.982203:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828825.982207:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982215:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828825.982219:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982226:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828825.982228:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982230:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.982232:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982234:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828825.982236:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828825.982238:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828825.982242:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828825.982244:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828825.982249:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828825.982257:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327282560:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828825.982262:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828825.982264:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828825.982268:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327282688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982274:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828825.982277:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828825.982281:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828825.982286:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828825.982289:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828825.982291:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828825.982294:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828825.982296:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:3.0:1787828825.982300:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828825.982302:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828825.982304:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:3.0:1787828825.982308:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828825.982310:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982312:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828825.982315:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327282816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982329:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828825.982330:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828825.982331:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828825.982334:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828825.982335:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828825.982336:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828825.982338:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828825.982339:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:3.0:1787828825.982342:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828825.982343:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828825.982344:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:3.0:1787828825.982347:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828825.982348:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982350:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828825.982353:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327282560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982359:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828825.982360:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828825.982362:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828825.982364:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828825.982365:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828825.982366:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828825.982368:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828825.982369:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:3.0:1787828825.982371:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828825.982372:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828825.982373:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:3.0:1787828825.982376:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828825.982377:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982380:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828825.982382:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828825.982383:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828825.982385:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828825.982389:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828825]  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828825.982400:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828825.982401:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828825.982403:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828825.982406:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828825.982407:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828825.982409:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828825.982412:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828825.982420:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982426:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828825.982428:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982429:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.982431:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828825.982434:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982440:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828825.982443:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982449:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828825.982451:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982452:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828825.982454:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982455:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828825.982457:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828825.982459:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828825.982462:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828825.982465:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828825.982469:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828825.982477:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327282944:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828825.982482:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982484:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828825.982486:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327282944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828825.982492:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828825.982494:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828825.982495:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828825.982498:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828825.982500:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828825.982501:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828825.982503:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828825.982504:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:3.0:1787828825.982507:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828825.982508:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828825.982510:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:3.0:1787828825.982513:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828825.982514:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982516:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828825.982523:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828825.982525:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828825.982527:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828825.982529:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828826.492771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828826.492779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828826.492781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828826.492786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828826.492789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828826.492793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828826.492796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828826.492798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828826.492801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828826.812888:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828826.812893:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828826.812898:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828827.516738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828827.516743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828827.516744:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828827.516748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828827.516751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828827.516754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828827.516756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828827.516758:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828827.516760:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828828.028812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828828.028822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828828.028824:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828828.028828:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828828.028833:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828828.028840:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2975
00000400:00000200:2.0:1787828828.028849:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828828.028857:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828828.028861:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828828.028865:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.028871:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828828.028873:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.028876:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828828.028880:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828828.028882:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828828.028884:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828828.028888:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828828.028890:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828828.028893:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828828.028896:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.028898:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828828.028899:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.028901:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828828.028904:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828828.028908:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828828.028911:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828828.028914:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828828.028918:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828828.028921:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.028924:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828828.028927:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828828.028933:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828828.028937:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828828.028941:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828828.028942:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.028945:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828828.028947:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828828.028951:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828828.028956:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787828828.028961:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828828.028968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828828.028980:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828828.028984:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828828.028987:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828828.029007:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828828.029010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828828.029012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828828.029013:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828828.029014:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828828.029017:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828828.029019:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2975
00000400:00000200:2.0:1787828828.029023:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828828.029027:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828828.029029:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828828.029032:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029035:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828828.029038:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029041:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828828.029043:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828828.029045:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828828.029047:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828828.029051:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828828.029053:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828828.029055:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828828.029059:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029062:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828828.029066:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029068:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828828.029071:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828828.029074:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029076:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828828.029080:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828828.029081:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787828828.029086:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828828.029092:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.029093:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828828.029097:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828828.029100:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:2.0:1787828828.029111:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828828.029118:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828828.029119:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828828.029122:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828828.029132:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828828.029140:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828828.029156:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787828828.029157:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787828828.029161:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.029164:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.029168:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828828.029172:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828828.029174:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.029176:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:2.0:1787828828.029179:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828828.029184:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787828828.029185:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.029199:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828828.029206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1787828828.029216:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327283072:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828828.029234:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828828.029241:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828828.029245:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.029252:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787828828.029258:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000010:3.0:1787828828.029266:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:2.0:1787828828.029272:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.029274:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828828.029295:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828828.029324:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828828.029332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828828.029334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828828.029341:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:3.0:1787828828.029343:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000100:00000001:2.0:1787828828.029344:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828828.029353:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828828.029361:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:3.0:1787828828.029362:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00080000:2.0:1787828828.029370:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2975
00000100:00000200:3.0:1787828828.029377:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283072, portal 25
00000400:00000200:2.0:1787828828.029380:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828828.029387:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828828.029393:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000400:00000200:2.0:1787828828.029394:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828828.029406:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828828.029416:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:3.0:1787828828.029418:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828828.029420:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828828.029428:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029441:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828828.029450:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787828828.029454:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1787828828.029456:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828828.029464:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1787828828.029464:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787828828.029474:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000010:3.0:1787828828.029476:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000020:00000001:2.0:1787828828.029490:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:3.0:1787828828.029492:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327283072, offset 0
00000020:00000001:2.0:1787828828.029499:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828828.029514:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:3.0:1787828828.029514:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029521:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828828.029526:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029528:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828828.029531:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828828.029533:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029536:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828828.029538:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828828.029542:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828828.029544:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.029547:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828828.029549:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828828.029552:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.029554:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828828.029560:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828828.029563:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828828.029566:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828828.029569:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.029571:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828828.029573:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828828.029575:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828828.029581:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828828.029585:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828828.029607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828828.029621:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828828.029624:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828828.029630:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787828828.029632:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787828828.029648:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828828.029652:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828828.029705:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787828828.029707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828828.029717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:3.0:1787828828.029722:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787828828.029725:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828828.029731:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828828.029739:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787828828.029740:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1787828828.029751:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2975
00000400:00000200:3.0:1787828828.029758:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828828.029768:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39983:39983:256:4294967295] 192.168.203.116@tcp LPNI seq info [39983:39983:8:4294967295]
00000400:00000200:3.0:1787828828.029774:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.029789:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787828828.029807:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:3.0:1787828828.029808:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.029826:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000400:00000200:2.0:1787828828.029827:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828828.029835:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828828.029852:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:3.0:1787828828.029853:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828828.029861:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787828828.029866:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1787828828.029873:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029882:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:3.1:1787828828.029892:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787828828.029893:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828828.029899:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000200:3.1:1787828828.029903:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1787828828.029905:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828828.029910:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828828.029917:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000100:3.0:1787828828.029923:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787828828.029926:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000010:3.0:1787828828.029933:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000100:00000040:2.0:1787828828.029940:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029947:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787828828.029950:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1787828828.029955:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828828.029961:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:3.0:1787828828.029970:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000040:2.0:1787828828.029971:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828828.029984:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:3.0:1787828828.029985:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000100:00000001:2.0:1787828828.029988:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828828.029992:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
02000000:00000001:2.0:1787828828.029994:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828828.030003:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828828.030007:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787828828.030008:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.030014:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828828.030018:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000200:3.0:1787828828.030024:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787828828.030026:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.030036:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828828.030047:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828828.030052:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:3.0:1787828828.030056:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787828828.030057:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828828.030064:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.030072:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1787828828.030073:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787828828.030075:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828828.030080:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828828.030090:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:3.0:1787828828.030097:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.030099:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030110:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:2.0:1787828828.030118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828828.030129:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.030131:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787828828.030142:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828828.030143:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030149:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297648813
00000100:00000040:2.0:1787828828.030157:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787828828.030166:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f780.
00000100:00000001:2.0:1787828828.030172:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030182:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787828828.030183:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (2985)
00000100:00000001:3.0:1787828828.030197:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297648813
00000100:00000010:3.0:1787828828.030205:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f180.
00000100:00000001:3.0:1787828828.030211:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828828.030220:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828828.030224:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030228:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828828.030235:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828828/real 1787828828]  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828828.030251:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828828.030253:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828828.030255:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828828.030258:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828828.030260:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828828.030263:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828828.030267:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828828.030275:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030282:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828828.030284:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030286:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.030289:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828828.030293:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030300:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828828.030305:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030312:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828828.030315:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030317:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.030318:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030321:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828828.030323:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828828.030327:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828828.030333:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828828.030335:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828828.030340:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828828.030348:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327283072:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828828.030353:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030355:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.030358:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327283072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030372:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828828.030375:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828828.030379:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828828.030383:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828828.030386:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828828.030388:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828828.030390:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828828.030393:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:3.0:1787828828.030397:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828828.030400:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828828.030402:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:3.0:1787828828.030406:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828828.030408:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030410:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030419:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828828.030421:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030424:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828828.030426:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828828.030428:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828828.030432:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030441:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828828.030443:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.030446:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828828.030448:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828828.030451:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828828.030455:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283200:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828828.030459:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828828.030463:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828828.030465:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.030466:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828828.030470:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828828.030472:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828828.030478:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828828.030480:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828828.030485:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828828.030490:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283200, portal 10
00000100:00000001:3.0:1787828828.030492:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828828.030494:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:3.0:1787828828.030501:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030508:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828828.030510:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828828.030513:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828828.030516:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327283200, offset 0
00000400:00000200:3.0:1787828828.030521:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828828.030526:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.030531:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828828.030535:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828828.030538:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828828.030542:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828828.030544:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828828.030548:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39984:39984:256:4294967295] 192.168.203.116@tcp LPNI seq info [39984:39984:8:4294967295]
00000400:00000200:3.0:1787828828.030553:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.030560:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828828.030565:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.030570:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828828.030574:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828828.030578:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828828.030583:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828828.030586:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828828.030603:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.030605:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828828.030609:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828828.030613:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828828.030617:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828828.030619:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828828.030622:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828828.030626:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030635:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.030639:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030645:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030646:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828828.030650:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828828.030683:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828828.030688:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828828.030691:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297648140
00000100:00000001:3.0:1787828828.030694:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297648813
00000100:00000001:3.0:1787828828.030696:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297648140
00000100:00000001:3.0:1787828828.030698:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297648813
00000100:00000001:3.0:1787828828.030699:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828828.030701:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030702:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828828.030706:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828828/real 1787828828]  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828828.030711:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828828.030712:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828828.030714:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828828.030716:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828828.030717:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828828.030719:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828828.030721:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828828.030727:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030731:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828828.030732:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030733:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.030734:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828828.030736:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030741:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828828.030744:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030748:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828828.030750:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030751:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.030752:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030753:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828828.030754:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828828.030756:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828828.030758:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828828.030760:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828828.030762:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828828.030767:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283200:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828828.030770:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828828.030773:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030778:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828828.030779:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.030780:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828828.030781:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828828.030782:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828828.030784:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283328:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828828.030786:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828828.030788:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828828.030789:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.030790:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828828.030793:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828828.030795:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828828.030798:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828828.030799:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828828.030802:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828828.030804:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283328, portal 4
00000100:00000001:3.0:1787828828.030806:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828828.030807:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:3.0:1787828828.030811:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030816:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828828.030817:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828828.030819:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828828.030820:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327283328, offset 0
00000400:00000200:3.0:1787828828.030823:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828828.030825:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.030828:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828828.030830:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828828.030832:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828828.030834:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828828.030835:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828828.030838:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39985:39985:256:4294967295] 192.168.203.116@tcp LPNI seq info [39985:39985:8:4294967295]
00000400:00000200:3.0:1787828828.030840:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.030845:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828828.030848:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.030850:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828828.030852:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828828.030854:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828828.030858:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828828.030860:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828828.030862:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.030864:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828828.030866:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828828.030868:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828828.030870:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828828.030872:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828828.030873:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828828.030876:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030881:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.030883:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030888:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030888:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828828.030890:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828828.030892:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828828.030894:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828828.030895:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297648147
00000100:00000001:3.0:1787828828.030897:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297648813
00000100:00000001:3.0:1787828828.030899:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297648147
00000100:00000001:3.0:1787828828.030901:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297648813
00000100:00000001:3.0:1787828828.030903:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828828.030914:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.030916:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828828.030918:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828828/real 1787828828]  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828828.030923:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828828.030923:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828828.030924:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828828.030925:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828828.030926:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828828.030928:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828828.030929:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828828.030935:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030940:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828828.030940:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030942:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.030942:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828828.030945:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030952:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828828.030956:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.030964:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828828.030966:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030968:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.030969:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.030972:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828828.030974:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828828.030976:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828828.030980:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828828.030983:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828828.030987:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828828.030996:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283328:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828828.031001:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828828.031004:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031009:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828828.031010:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.031012:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828828.031014:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828828.031015:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828828.031017:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283456:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828828.031019:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828828.031020:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828828.031033:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828828.031035:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828828.031037:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828828.031039:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828828.031041:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828828.031043:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828828.031046:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828828.031049:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283456, portal 4
00000100:00000001:3.0:1787828828.031051:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828828.031053:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:3.0:1787828828.031059:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031067:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828828.031069:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828828.031072:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828828.031075:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327283456, offset 0
00000400:00000200:3.0:1787828828.031079:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828828.031084:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.031087:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828828.031091:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828828.031093:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828828.031096:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828828.031099:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828828.031102:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39986:39986:256:4294967295] 192.168.203.116@tcp LPNI seq info [39986:39986:8:4294967295]
00000400:00000200:3.0:1787828828.031106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828828.031111:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828828.031114:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.031117:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828828.031120:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828828.031122:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828828.031124:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828828.031126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828828.031129:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828828.031131:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828828.031133:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828828.031135:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828828.031137:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828828.031139:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828828.031141:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828828.031144:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031149:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.031151:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828853 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031161:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.031162:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828828.031164:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828828.031166:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828828.031168:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828828.031170:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297648152
00000100:00000001:3.0:1787828828.031172:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297648813
00000100:00000001:3.0:1787828828.031173:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297648152
00000100:00000001:3.0:1787828828.031175:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297648813
00000100:00000001:3.0:1787828828.031177:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828828.031178:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.031179:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828828.031181:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828828/real 1787828828]  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828828.031186:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828828.031187:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828828.031187:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828828.031189:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828828.031190:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828828.031192:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828828.031194:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828828.031199:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031203:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828828.031204:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031205:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.031206:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828828.031208:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031213:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828828.031216:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031220:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828828.031221:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031222:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828828.031223:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031224:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828828.031225:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828828.031227:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828828.031229:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828828.031230:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828828.031233:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828828.031237:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283456:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828828.031239:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031241:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.031242:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327283200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031247:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828828.031248:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828828.031250:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828828.031251:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828828.031253:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828828.031254:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828828.031255:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828828.031257:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:3.0:1787828828.031259:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828828.031260:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828828.031261:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:3.0:1787828828.031263:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828828.031265:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031266:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.031268:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327283328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031272:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828828.031273:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828828.031274:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828828.031275:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828828.031276:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828828.031277:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828828.031278:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828828.031279:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828828.031280:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828828.031281:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828828.031281:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:3.0:1787828828.031283:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828828.031284:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031285:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828828.031286:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327283456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828853 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828828.031290:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828828.031291:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828828.031292:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828828.031293:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828828.031293:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828828.031294:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828828.031295:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828828.031296:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:3.0:1787828828.031297:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828828.031298:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828828.031299:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:3.0:1787828828.031300:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828828.031301:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031303:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828828.031305:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828828.031307:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828828.031309:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828828.031311:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828828.540796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828828.540805:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828828.540808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828828.540814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828828.540818:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828828.540823:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828828.540827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828828.540830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828828.540834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828828.732719:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297648813, at 4297648832
00000100:00000010:3.0:1787828828.732731:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f180.
00000100:00000040:2.0:1787828828.732769:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297648813, at 4297648832
00000100:00000010:2.0:1787828828.732783:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f780.
00000020:00000001:2.0:1787828828.860851:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828828.860857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828828.860873:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828829.564786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828829.564795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828829.564797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828829.564804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828829.564808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828829.564811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828829.564816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828829.564819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828829.564823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828829.884830:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828829.884835:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828829.884839:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828830.588750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828830.588761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828830.588764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828830.588773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828830.588778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828830.588783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828830.588787:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828830.588792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828830.588796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828831.612804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828831.612814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828831.612816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828831.612831:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828831.612835:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828831.612840:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828831.612843:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828831.612846:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828831.612849:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828831.932884:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828831.932890:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828831.932896:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828832.636766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828832.636775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828832.636777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828832.636783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828832.636786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828832.636790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828832.636793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828832.636795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828832.636798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828832.956876:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828832.956882:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828832.956887:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828833.149070:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828833.149075:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828833.149079:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828833.149088:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149101:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828833.149104:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.149108:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828833.149112:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828833.149118:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828833.149125:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327283584:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828833.149132:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828833.149136:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828833.149140:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.149146:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828833.149153:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828833.149155:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828833.149164:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828833.149168:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828833.149175:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828833.149182:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283584, portal 25
00000100:00000001:3.0:1787828833.149186:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828833.149187:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:3.0:1787828833.149196:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149209:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828833.149214:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828833.149219:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828833.149224:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327283584, offset 0
00000400:00000200:3.0:1787828833.149233:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828833.149242:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.149251:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828833.149261:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828833.149266:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828833.149271:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828833.149272:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828833.149275:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39987:39987:256:4294967295] 192.168.203.116@tcp LPNI seq info [39987:39987:8:4294967295]
00000400:00000200:3.0:1787828833.149278:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.149291:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828833.149305:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.149315:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828833.149326:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828833.149333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828833.149346:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828833.149353:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828833.149362:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.149365:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828833.149375:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828833.149381:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828833.149386:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828833.149389:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828833.149395:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828833.149402:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149415:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.149421:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149433:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.149438:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828833.149444:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828833.149451:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828833.149454:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297653932
00000100:00000010:3.0:1787828833.149460:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f180.
00000100:00000001:3.0:1787828833.149468:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828833.149475:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297653932
00000100:00000010:3.0:1787828833.149481:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f060.
00000100:00000001:3.0:1787828833.149487:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828833.149491:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828833.149495:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.149498:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828833.149504:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828833/real 1787828833]  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828833.149517:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828833.149520:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828833.149523:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828833.149526:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828833.149529:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828833.149533:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828833.149538:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828833.149549:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149575:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828833.149577:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.149581:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.149584:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828833.149592:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149603:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828833.149610:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149631:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828833.149634:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.149637:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.149639:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.149642:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828833.149647:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828833.149652:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828833.149694:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828833.149698:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828833.149706:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828833.149720:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327283584:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828833.149727:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.149731:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.149737:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327283584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149749:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828833.149752:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828833.149755:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828833.149760:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828833.149763:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828833.149766:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828833.149770:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828833.149774:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:3.0:1787828833.149780:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828833.149783:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828833.149787:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:3.0:1787828833.149793:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828833.149797:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.149800:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.149808:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828833.149811:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.149815:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828833.149819:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828833.149821:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828833.149827:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149837:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828833.149840:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.149843:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828833.149846:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828833.149849:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828833.149854:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283712:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828833.149862:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828833.149864:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828833.149867:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.149872:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828833.149875:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828833.149880:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828833.149887:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828833.149891:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828833.149897:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828833.149901:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283712, portal 10
00000100:00000001:3.0:1787828833.149905:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828833.149907:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:3.0:1787828833.149914:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.149922:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828833.149925:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828833.149929:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828833.149933:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327283712, offset 0
00000400:00000200:3.0:1787828833.149942:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828833.149955:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.149961:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828833.149967:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828833.149972:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828833.149976:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828833.149981:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828833.149985:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39988:39988:256:4294967295] 192.168.203.116@tcp LPNI seq info [39988:39988:8:4294967295]
00000400:00000200:3.0:1787828833.149990:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.149999:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828833.150003:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.150006:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828833.150009:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828833.150011:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828833.150015:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828833.150030:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828833.150033:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.150034:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828833.150037:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828833.150039:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828833.150042:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828833.150043:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828833.150045:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828833.150048:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150054:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.150056:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150060:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150061:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828833.150064:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828833.150066:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828833.150069:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828833.150070:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297653259
00000100:00000001:3.0:1787828833.150073:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297653932
00000100:00000001:3.0:1787828833.150074:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297653259
00000100:00000001:3.0:1787828833.150076:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297653932
00000100:00000001:3.0:1787828833.150078:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828833.150079:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150080:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828833.150082:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828833/real 1787828833]  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828833.150087:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828833.150088:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828833.150089:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828833.150091:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828833.150092:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828833.150093:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828833.150095:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828833.150100:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150104:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828833.150105:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150106:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.150107:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828833.150109:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150113:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828833.150115:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150120:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828833.150122:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150124:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.150125:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150126:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828833.150127:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828833.150129:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828833.150131:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828833.150132:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828833.150135:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828833.150139:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283712:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828833.150142:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828833.150144:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150148:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828833.150149:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.150150:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828833.150151:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828833.150152:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828833.150154:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828833.150156:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828833.150158:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828833.150159:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.150160:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828833.150162:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828833.150164:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828833.150166:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828833.150167:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828833.150169:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828833.150171:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283840, portal 4
00000100:00000001:3.0:1787828833.150173:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828833.150174:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:3.0:1787828833.150178:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150183:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828833.150184:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828833.150185:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828833.150187:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327283840, offset 0
00000400:00000200:3.0:1787828833.150189:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828833.150192:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.150194:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828833.150196:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828833.150197:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828833.150199:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828833.150200:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828833.150203:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39989:39989:256:4294967295] 192.168.203.116@tcp LPNI seq info [39989:39989:8:4294967295]
00000400:00000200:3.0:1787828833.150205:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.150209:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828833.150212:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.150214:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828833.150216:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828833.150218:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828833.150221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828833.150222:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828833.150225:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.150226:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828833.150228:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828833.150230:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828833.150232:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828833.150233:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828833.150235:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828833.150237:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150242:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.150244:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150248:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150249:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828833.150251:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828833.150253:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828833.150254:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828833.150256:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297653266
00000100:00000001:3.0:1787828833.150257:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297653932
00000100:00000001:3.0:1787828833.150259:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297653266
00000100:00000001:3.0:1787828833.150260:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297653932
00000100:00000001:3.0:1787828833.150261:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828833.150263:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150264:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828833.150266:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828833/real 1787828833]  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828833.150270:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828833.150271:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828833.150272:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828833.150273:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828833.150274:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828833.150275:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828833.150277:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828833.150282:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150286:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828833.150287:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150288:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.150289:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828833.150291:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150297:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828833.150301:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150308:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828833.150311:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150312:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.150314:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150316:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828833.150317:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828833.150320:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828833.150323:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828833.150326:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828833.150330:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828833.150339:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828833.150343:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150345:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.150348:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327283712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150353:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828833.150355:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828833.150356:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828833.150358:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828833.150359:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828833.150360:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828833.150362:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828833.150363:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:3.0:1787828833.150365:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828833.150366:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828833.150367:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:3.0:1787828833.150369:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828833.150370:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150371:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.150373:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327283840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150377:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828833.150378:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828833.150379:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828833.150380:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828833.150381:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828833.150381:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828833.150383:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828833.150383:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:3.0:1787828833.150385:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828833.150385:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828833.150386:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:3.0:1787828833.150387:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828833.150388:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150389:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150392:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828833.150393:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150394:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828833.150395:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828833.150396:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828833.150398:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150402:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828833.150403:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.150404:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828833.150405:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828833.150407:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828833.150408:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828833.150411:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828833.150412:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828833.150413:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828833.150414:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828833.150417:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828833.150418:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828833.150420:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828833.150421:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828833.150424:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828833.150426:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327283968, portal 4
00000100:00000001:3.0:1787828833.150427:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828833.150428:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:3.0:1787828833.150432:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150436:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828833.150438:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828833.150439:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828833.150441:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327283968, offset 0
00000400:00000200:3.0:1787828833.150443:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828833.150445:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.150448:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828833.150450:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828833.150451:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828833.150453:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828833.150455:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828833.150457:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39990:39990:256:4294967295] 192.168.203.116@tcp LPNI seq info [39990:39990:8:4294967295]
00000400:00000200:3.0:1787828833.150460:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828833.150463:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828833.150466:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.150468:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828833.150471:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828833.150473:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787828833.150476:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828833.150478:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787828833.150480:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828833.150481:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828833.150483:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828833.150486:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828833.150487:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828833.150489:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828833.150490:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828833.150492:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150497:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.150499:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828858 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150503:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150504:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828833.150506:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828833.150508:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828833.150510:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828833.150511:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297653271
00000100:00000001:3.0:1787828833.150513:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297653932
00000100:00000001:3.0:1787828833.150514:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297653271
00000100:00000001:3.0:1787828833.150516:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297653932
00000100:00000001:3.0:1787828833.150517:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828833.150519:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150520:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828833.150522:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828833/real 1787828833]  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828833.150527:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828833.150527:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828833.150529:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828833.150530:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828833.150531:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828833.150533:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828833.150535:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828833.150539:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150543:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828833.150544:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150545:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.150546:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828833.150548:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150552:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828833.150571:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150576:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828833.150578:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150579:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828833.150580:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150581:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828833.150582:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828833.150584:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828833.150586:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828833.150588:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828833.150590:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828833.150596:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327283968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828833.150598:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150600:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828833.150602:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327283968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828858 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828833.150606:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828833.150607:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828833.150608:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828833.150609:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828833.150610:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828833.150611:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828833.150612:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828833.150613:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:3.0:1787828833.150615:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828833.150616:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828833.150617:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:3.0:1787828833.150618:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828833.150619:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150620:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828833.150622:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828833.150623:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828833.150624:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828833.150626:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828833.660751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828833.660760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828833.660762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828833.660767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828833.660770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828833.660774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828833.660777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828833.660779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828833.660781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828833.852843:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297653932, at 4297653952
00000100:00000040:2.0:1787828833.852871:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297653932, at 4297653952
00000100:00000010:3.0:1787828833.852875:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f060.
00000100:00000010:2.0:1787828833.852908:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f180.
00000100:00000001:2.0:1787828834.684741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828834.684749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828834.684751:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828834.684757:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828834.684760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828834.684764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828834.684766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828834.684769:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828834.684771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828835.004892:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828835.004898:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828835.004903:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828835.708812:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828835.708822:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828835.708825:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828835.708834:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828835.708839:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828835.708844:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828835.708848:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828835.708851:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828835.708855:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828836.028891:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828836.028897:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828836.028901:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828836.732758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828836.732768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828836.732771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828836.732777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828836.732782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828836.732786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828836.732789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828836.732792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828836.732795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828837.756800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828837.756812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828837.756814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828837.756819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828837.756823:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828837.756827:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828837.756830:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828837.756832:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828837.756836:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828838.076819:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828838.076826:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828838.076831:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787828838.268951:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828838.268957:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828838.268962:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828838.268973:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.268989:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828838.268992:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.268997:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828838.269001:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828838.269005:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828838.269014:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327284096:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828838.269025:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828838.269032:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828838.269037:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.269044:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828838.269054:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828838.269061:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828838.269086:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828838.269097:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828838.269123:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828838.269134:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284096, portal 25
00000100:00000001:3.0:1787828838.269138:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828838.269141:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:3.0:1787828838.269153:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269172:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828838.269180:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828838.269187:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828838.269194:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327284096, offset 0
00000400:00000200:3.0:1787828838.269204:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828838.269217:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.269226:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828838.269234:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828838.269242:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828838.269247:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828838.269251:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828838.269258:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39991:39991:256:4294967295] 192.168.203.116@tcp LPNI seq info [39991:39991:8:4294967295]
00000400:00000200:3.0:1787828838.269267:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.269281:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828838.269291:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.269297:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828838.269305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828838.269311:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828838.269322:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828838.269327:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787828838.269333:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.269337:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828838.269346:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828838.269353:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828838.269362:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828838.269367:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828838.269374:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828838.269384:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269429:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.269441:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269457:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.269463:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828838.269473:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828838.269484:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828838.269489:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297659052
00000100:00000010:3.0:1787828838.269498:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f060.
00000100:00000001:3.0:1787828838.269508:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828838.269515:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297659052
00000100:00000010:3.0:1787828838.269539:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fde0.
00000100:00000001:3.0:1787828838.269546:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828838.269553:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828838.269574:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.269577:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828838.269584:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828838/real 1787828838]  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828838.269606:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828838.269609:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828838.269614:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828838.269619:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828838.269623:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828838.269629:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828838.269635:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828838.269642:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269648:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828838.269649:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269651:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.269652:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828838.269655:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269660:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828838.269663:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269668:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828838.269670:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269671:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.269672:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269702:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828838.269704:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828838.269706:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828838.269709:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828838.269710:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828838.269714:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828838.269720:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327284096:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828838.269725:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269728:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.269731:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327284096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269737:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828838.269740:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828838.269743:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828838.269746:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828838.269749:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828838.269750:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828838.269753:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828838.269755:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:3.0:1787828838.269759:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828838.269761:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828838.269763:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:3.0:1787828838.269767:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828838.269768:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269771:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269777:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828838.269779:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.269780:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828838.269782:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828838.269790:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828838.269793:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269798:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828838.269799:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.269800:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828838.269802:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828838.269803:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828838.269806:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284224:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828838.269808:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828838.269809:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828838.269811:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.269812:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828838.269815:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828838.269816:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828838.269819:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828838.269821:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828838.269823:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828838.269826:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284224, portal 10
00000100:00000001:3.0:1787828838.269827:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828838.269828:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:3.0:1787828838.269832:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269837:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828838.269839:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828838.269840:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828838.269842:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327284224, offset 0
00000400:00000200:3.0:1787828838.269844:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828838.269847:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.269850:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828838.269852:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828838.269854:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828838.269856:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828838.269862:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828838.269864:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39992:39992:256:4294967295] 192.168.203.116@tcp LPNI seq info [39992:39992:8:4294967295]
00000400:00000200:3.0:1787828838.269867:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.269872:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828838.269875:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.269877:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828838.269880:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828838.269882:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828838.269885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828838.269887:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787828838.269889:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.269891:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828838.269893:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828838.269896:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828838.269898:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828838.269899:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828838.269901:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828838.269903:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269908:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.269910:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269932:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.269933:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828838.269936:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828838.269939:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828838.269941:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828838.269943:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297658379
00000100:00000001:3.0:1787828838.269946:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297659052
00000100:00000001:3.0:1787828838.269947:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297658379
00000100:00000001:3.0:1787828838.269949:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297659052
00000100:00000001:3.0:1787828838.269951:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828838.269952:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.269954:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828838.269956:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828838/real 1787828838]  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828838.269962:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828838.269963:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828838.269964:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828838.269966:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828838.269967:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828838.269969:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828838.269971:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828838.269977:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269982:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828838.269983:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.269984:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.269985:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828838.269988:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.269992:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828838.269995:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270000:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828838.270001:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270003:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.270003:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270005:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828838.270006:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828838.270008:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828838.270010:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828838.270012:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828838.270015:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828838.270023:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284224:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828838.270028:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828838.270032:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270047:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828838.270048:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.270051:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828838.270053:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828838.270056:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828838.270060:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828838.270064:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828838.270066:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828838.270068:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.270071:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828838.270075:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828838.270077:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828838.270080:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828838.270082:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828838.270085:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828838.270087:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284352, portal 4
00000100:00000001:3.0:1787828838.270089:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828838.270090:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:3.0:1787828838.270111:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270115:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828838.270116:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828838.270118:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828838.270120:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327284352, offset 0
00000400:00000200:3.0:1787828838.270122:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828838.270125:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.270127:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828838.270129:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828838.270131:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828838.270133:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828838.270134:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828838.270137:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39993:39993:256:4294967295] 192.168.203.116@tcp LPNI seq info [39993:39993:8:4294967295]
00000400:00000200:3.0:1787828838.270140:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.270144:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828838.270147:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.270151:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828838.270155:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828838.270174:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828838.270178:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828838.270181:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787828838.270186:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.270188:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828838.270192:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828838.270196:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828838.270199:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828838.270201:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828838.270205:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828838.270210:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270219:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.270223:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270228:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.270230:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828838.270233:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828838.270235:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828838.270238:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828838.270239:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297658386
00000100:00000001:3.0:1787828838.270241:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297659052
00000100:00000001:3.0:1787828838.270243:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297658386
00000100:00000001:3.0:1787828838.270245:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297659052
00000100:00000001:3.0:1787828838.270246:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828838.270248:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.270250:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828838.270252:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828838/real 1787828838]  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828838.270259:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828838.270259:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828838.270261:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828838.270262:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828838.270263:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828838.270265:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828838.270267:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828838.270274:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270278:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828838.270279:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270280:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.270281:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828838.270284:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270288:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828838.270291:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270312:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828838.270314:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270315:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.270316:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270317:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828838.270318:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828838.270320:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828838.270322:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828838.270323:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828838.270326:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828838.270331:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828838.270333:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828838.270335:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270339:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828838.270340:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.270341:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828838.270343:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828838.270344:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828838.270345:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284480:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828838.270347:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828838.270348:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828838.270349:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828838.270351:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828838.270353:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828838.270354:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828838.270356:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828838.270358:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828838.270376:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828838.270379:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284480, portal 4
00000100:00000001:3.0:1787828838.270381:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828838.270382:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:3.0:1787828838.270389:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270396:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828838.270399:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828838.270402:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828838.270405:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327284480, offset 0
00000400:00000200:3.0:1787828838.270410:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828838.270415:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.270419:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828838.270423:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828838.270427:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828838.270429:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828838.270432:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828838.270436:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39994:39994:256:4294967295] 192.168.203.116@tcp LPNI seq info [39994:39994:8:4294967295]
00000400:00000200:3.0:1787828838.270439:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828838.270444:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828838.270448:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.270451:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828838.270454:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828838.270456:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787828838.270459:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828838.270461:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787828838.270464:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828838.270465:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828838.270468:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828838.270471:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828838.270473:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828838.270474:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828838.270476:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828838.270479:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270485:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.270487:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270492:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.270494:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828838.270496:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828838.270499:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828838.270501:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828838.270503:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297658391
00000100:00000001:3.0:1787828838.270504:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297659052
00000100:00000001:3.0:1787828838.270506:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297658391
00000100:00000001:3.0:1787828838.270508:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297659052
00000100:00000001:3.0:1787828838.270509:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828838.270511:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.270512:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828838.270515:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828838/real 1787828838]  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828838.270534:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828838.270536:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828838.270537:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828838.270539:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828838.270540:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828838.270542:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828838.270545:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828838.270551:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270572:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828838.270573:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270574:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.270575:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828838.270577:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270581:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828838.270584:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270589:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828838.270590:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270591:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828838.270592:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270593:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828838.270594:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828838.270596:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828838.270598:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828838.270599:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828838.270601:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828838.270606:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284480:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828838.270608:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270609:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.270611:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327284224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270615:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828838.270616:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828838.270617:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828838.270619:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828838.270620:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828838.270621:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828838.270622:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828838.270623:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:3.0:1787828838.270625:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828838.270626:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828838.270627:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:3.0:1787828838.270629:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828838.270630:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270631:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.270634:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327284352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270638:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828838.270639:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828838.270639:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828838.270641:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828838.270642:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828838.270643:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828838.270644:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828838.270644:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:3.0:1787828838.270645:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828838.270646:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828838.270647:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:3.0:1787828838.270648:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828838.270649:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270650:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828838.270652:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327284480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828838.270655:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828838.270656:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828838.270657:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828838.270658:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828838.270659:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828838.270659:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828838.270660:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828838.270661:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:3.0:1787828838.270662:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828838.270663:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828838.270664:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:3.0:1787828838.270665:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828838.270666:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270667:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828838.270669:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828838.270670:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828838.270671:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828838.270692:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828838.780706:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828838.780712:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828838.780714:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828838.780718:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828838.780721:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828838.780723:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828838.780725:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828838.780727:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828838.780729:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828838.972742:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297659052, at 4297659072
00000100:00000010:3.0:1787828838.972761:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fde0.
00000100:00000040:2.0:1787828838.972762:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297659052, at 4297659072
00000100:00000010:2.0:1787828838.972775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f060.
00000020:00000001:2.0:1787828839.100835:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828839.100838:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828839.100841:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828839.804777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828839.804783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828839.804785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828839.804789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828839.804792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828839.804795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828839.804797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828839.804799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828839.804801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828840.828749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828840.828754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828840.828756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828840.828760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828840.828763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828840.828766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828840.828768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828840.828770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828840.828772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828841.148875:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828841.148879:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828841.148882:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828841.852752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828841.852758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828841.852759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828841.852763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828841.852767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828841.852770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828841.852772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828841.852774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828841.852776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828842.172818:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828842.172821:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828842.172824:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828842.876741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828842.876747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828842.876749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828842.876754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828842.876758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828842.876761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828842.876764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828842.876767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828842.876770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828843.388816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828843.388824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828843.388826:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828843.388831:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828843.388836:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828843.388842:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2991
00000400:00000200:2.0:1787828843.388850:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828843.388855:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828843.388858:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828843.388860:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.388862:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.388864:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.388867:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828843.388870:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828843.388872:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828843.388874:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828843.388877:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828843.388879:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828843.388881:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828843.388883:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.388884:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.388885:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.388886:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828843.388889:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828843.388891:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828843.388893:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828843.388895:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828843.388899:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828843.388901:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.388902:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828843.388904:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828843.388913:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828843.388915:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828843.388918:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828843.388919:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.388922:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828843.388925:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828843.388929:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828843.388938:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828843.388945:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828843.388954:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828843.388966:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828843.388970:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828843.388977:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828843.388998:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828843.389000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828843.389003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828843.389004:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828843.389005:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828843.389009:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828843.389011:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2991
00000400:00000200:2.0:1787828843.389014:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828843.389016:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828843.389018:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828843.389020:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389022:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.389024:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389025:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828843.389027:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828843.389028:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828843.389030:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828843.389032:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828843.389033:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828843.389035:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828843.389037:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389038:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.389040:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389042:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828843.389044:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828843.389046:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389048:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828843.389049:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828843.389052:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787828843.389055:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828843.389059:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389059:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828843.389064:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828843.389065:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:2.0:1787828843.389068:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828843.389071:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.389075:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828843.389077:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787828843.389078:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828843.389082:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828843.389086:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828843.389089:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.389091:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828843.389092:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828843.389094:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828843.389096:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828843.389100:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828843.389109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828843.389111:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828843.389115:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.389121:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828843.389123:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.389123:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828843.389129:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787828843.389133:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787828843.389135:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828843.389147:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327284608:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828843.389148:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828843.389153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787828843.389157:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787828843.389158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828843.389162:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828843.389163:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828843.389165:0:37:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:3.0:1787828843.389165:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828843.389170:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828843.389171:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2991
00000400:00000200:2.0:1787828843.389178:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787828843.389178:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:2.0:1787828843.389181:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828843.389184:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828843.389186:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389187:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:3.0:1787828843.389188:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
00000100:00000040:2.0:1787828843.389189:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389195:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:3.0:1787828843.389195:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828843.389202:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828843.389203:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828843.389204:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828843.389205:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828843.389208:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:3.0:1787828843.389208:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787828843.389214:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:3.0:1787828843.389216:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000100:00000040:2.0:1787828843.389217:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389221:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.389224:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389225:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828843.389227:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000010:3.0:1787828843.389228:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00080000:2.0:1787828843.389230:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389235:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828843.389236:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828843.389238:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828843.389240:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828843.389241:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284608, portal 25
00000100:00000001:2.0:1787828843.389242:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828843.389246:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787828843.389247:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000008:00000001:2.0:1787828843.389249:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389252:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000100:00000001:2.0:1787828843.389253:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828843.389258:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828843.389261:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828843.389265:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828843.389266:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828843.389266:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828843.389270:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828843.389271:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828843.389274:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828843.389279:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828843.389284:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389287:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:2.0:1787828843.389291:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787828843.389295:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389303:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1787828843.389303:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000001:2.0:1787828843.389308:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828843.389310:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327284608, offset 0
00000100:00000040:2.0:1787828843.389312:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828843.389320:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828843.389323:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787828843.389324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828843.389330:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828843.389331:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828843.389333:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828843.389334:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828843.389338:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2991
00000400:00000200:3.0:1787828843.389339:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.389349:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828843.389355:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828843.389360:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828843.389364:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828843.389367:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828843.389371:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39995:39995:256:4294967295] 192.168.203.116@tcp LPNI seq info [39995:39995:8:4294967295]
00000400:00000200:3.0:1787828843.389375:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.389385:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787828843.389392:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:3.0:1787828843.389394:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787828843.389399:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828843.389401:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828843.389404:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000010:3.0:1787828843.389405:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000100:00000001:2.0:1787828843.389406:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.389410:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389412:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828843.389414:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828843.389416:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828843.389419:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.0:1787828843.389420:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787828843.389421:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828843.389424:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828843.389428:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:3.0:1787828843.389429:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000040:2.0:1787828843.389432:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389437:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828843.389440:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389443:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:3.1:1787828843.389446:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787828843.389447:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828843.389452:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:3.1:1787828843.389455:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:2.0:1787828843.389456:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828843.389459:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828843.389462:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828843.389466:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787828843.389468:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787828843.389469:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828843.389472:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000010:3.0:1787828843.389474:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000008:00000001:2.0:1787828843.389475:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.389479:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828843.389496:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828843.389499:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000200:3.0:1787828843.389501:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787828843.389502:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828843.389504:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.389508:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828843.389509:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828843.389512:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828843.389514:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000010:2.0:1787828843.389518:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000400:00000200:3.0:1787828843.389521:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000100:00000001:2.0:1787828843.389522:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787828843.389528:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00100000:2.0:1787828843.389531:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828843.389535:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787828843.389540:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828843.389542:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787828843.389545:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828843.389548:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828843.389579:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389583:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787828843.389584:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3001)
00000100:00000040:3.0:1787828843.389591:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389601:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389603:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828843.389611:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828843.389615:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828843.389619:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297664172
00000100:00000010:3.0:1787828843.389624:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fde0.
00000100:00000001:3.0:1787828843.389631:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828843.389636:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297664172
00000100:00000010:3.0:1787828843.389639:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f300.
00000100:00000001:3.0:1787828843.389642:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828843.389645:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828843.389648:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389651:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828843.389686:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843]  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828843.389697:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828843.389699:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828843.389701:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828843.389704:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828843.389706:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828843.389708:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828843.389712:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828843.389721:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389729:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828843.389731:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.389733:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.389736:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828843.389740:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389748:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828843.389753:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389761:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828843.389764:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.389766:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.389768:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.389770:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828843.389773:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828843.389776:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828843.389779:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828843.389782:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828843.389787:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828843.389797:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327284608:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828843.389802:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.389804:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.389808:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327284608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389815:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828843.389818:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828843.389821:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828843.389824:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828843.389826:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828843.389828:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828843.389830:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828843.389832:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:3.0:1787828843.389836:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828843.389838:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828843.389840:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:3.0:1787828843.389843:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828843.389844:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.389847:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.389853:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828843.389855:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.389857:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828843.389859:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828843.389861:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828843.389865:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389872:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828843.389874:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.389876:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828843.389878:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828843.389880:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828843.389884:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284736:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828843.389887:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828843.389889:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828843.389891:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.389893:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828843.389896:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828843.389899:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828843.389903:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828843.389905:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828843.389909:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828843.389912:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284736, portal 10
00000100:00000001:3.0:1787828843.389915:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828843.389916:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:3.0:1787828843.389922:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.389929:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828843.389932:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828843.389934:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828843.389937:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327284736, offset 0
00000400:00000200:3.0:1787828843.389941:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828843.389946:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.389950:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828843.389953:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828843.389956:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828843.389959:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828843.389962:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828843.389965:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39996:39996:256:4294967295] 192.168.203.116@tcp LPNI seq info [39996:39996:8:4294967295]
00000400:00000200:3.0:1787828843.389970:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.389976:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828843.389981:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828843.389984:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828843.389988:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828843.389992:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828843.389996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828843.389999:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828843.390004:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828843.390006:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828843.390009:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828843.390013:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828843.390016:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828843.390018:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828843.390021:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828843.390024:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390032:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.390036:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390043:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390044:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828843.390048:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828843.390051:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828843.390055:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828843.390057:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297663499
00000100:00000001:3.0:1787828843.390060:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297664172
00000100:00000001:3.0:1787828843.390062:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297663499
00000100:00000001:3.0:1787828843.390064:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297664172
00000100:00000001:3.0:1787828843.390067:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828843.390069:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390071:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828843.390075:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843]  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828843.390083:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828843.390085:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828843.390086:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828843.390089:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828843.390091:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828843.390093:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828843.390096:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828843.390105:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390111:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828843.390113:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390115:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.390116:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828843.390120:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390128:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828843.390132:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390139:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828843.390142:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390144:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.390145:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390147:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828843.390149:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828843.390153:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828843.390156:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828843.390158:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828843.390162:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828843.390170:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284736:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828843.390174:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828843.390177:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390184:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828843.390185:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.390188:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828843.390189:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828843.390191:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828843.390194:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828843.390197:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828843.390199:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828843.390201:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.390203:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828843.390206:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828843.390209:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828843.390212:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828843.390215:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828843.390219:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828843.390222:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284864, portal 4
00000100:00000001:3.0:1787828843.390224:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828843.390225:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:3.0:1787828843.390231:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390244:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828843.390247:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828843.390249:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828843.390252:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327284864, offset 0
00000400:00000200:3.0:1787828843.390256:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828843.390260:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.390274:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828843.390277:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828843.390279:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828843.390285:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828843.390287:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828843.390291:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39997:39997:256:4294967295] 192.168.203.116@tcp LPNI seq info [39997:39997:8:4294967295]
00000400:00000200:3.0:1787828843.390295:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.390301:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828843.390320:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828843.390323:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828843.390327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828843.390330:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828843.390334:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828843.390337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828843.390341:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828843.390343:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828843.390347:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828843.390350:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828843.390353:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828843.390355:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828843.390357:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828843.390361:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390369:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.390372:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390401:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390402:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828843.390405:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828843.390408:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828843.390411:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828843.390414:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297663506
00000100:00000001:3.0:1787828843.390417:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297664172
00000100:00000001:3.0:1787828843.390419:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297663506
00000100:00000001:3.0:1787828843.390421:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297664172
00000100:00000001:3.0:1787828843.390423:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828843.390425:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390427:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828843.390431:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843]  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828843.390438:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828843.390440:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828843.390441:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828843.390444:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828843.390446:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828843.390447:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828843.390451:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828843.390459:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390465:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828843.390467:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390468:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.390470:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828843.390473:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390480:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828843.390499:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390506:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828843.390509:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390511:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.390512:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390515:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828843.390516:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828843.390519:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828843.390522:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828843.390525:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828843.390528:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828843.390537:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828843.390541:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828843.390544:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390551:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828843.390552:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.390554:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828843.390556:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828843.390558:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828843.390561:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828843.390564:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828843.390565:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828843.390567:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828843.390570:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828843.390573:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828843.390575:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828843.390578:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828843.390580:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828843.390583:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828843.390586:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327284992, portal 4
00000100:00000001:3.0:1787828843.390588:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828843.390589:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:3.0:1787828843.390594:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390600:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828843.390602:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828843.390605:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828843.390607:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327284992, offset 0
00000400:00000200:3.0:1787828843.390611:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828843.390615:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.390618:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828843.390621:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828843.390624:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828843.390626:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828843.390628:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828843.390631:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39998:39998:256:4294967295] 192.168.203.116@tcp LPNI seq info [39998:39998:8:4294967295]
00000400:00000200:3.0:1787828843.390635:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828843.390641:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828843.390645:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828843.390648:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828843.390652:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828843.390672:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787828843.390678:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828843.390681:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787828843.390685:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828843.390688:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828843.390691:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828843.390695:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828843.390698:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828843.390700:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828843.390702:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828843.390705:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390713:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.390716:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390722:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390724:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828843.390726:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828843.390745:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828843.390748:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828843.390750:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297663512
00000100:00000001:3.0:1787828843.390753:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297664172
00000100:00000001:3.0:1787828843.390755:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297663512
00000100:00000001:3.0:1787828843.390757:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297664172
00000100:00000001:3.0:1787828843.390759:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828843.390761:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390762:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828843.390765:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843]  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828843.390773:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828843.390774:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828843.390776:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828843.390778:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828843.390779:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828843.390782:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828843.390785:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828843.390792:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390799:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828843.390800:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390802:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.390803:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828843.390807:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390814:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828843.390817:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390825:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828843.390828:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390829:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828843.390830:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390832:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828843.390834:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828843.390837:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828843.390840:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828843.390842:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828843.390846:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828843.390854:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327284992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828843.390858:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390860:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.390864:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327284736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390870:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828843.390872:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828843.390874:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828843.390877:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828843.390878:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828843.390880:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828843.390882:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828843.390884:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:3.0:1787828843.390886:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828843.390888:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828843.390889:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:3.0:1787828843.390892:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828843.390893:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390895:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.390898:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327284864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390906:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828843.390908:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828843.390909:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828843.390912:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828843.390913:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828843.390914:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828843.390917:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828843.390918:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828843.390920:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828843.390922:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828843.390923:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:3.0:1787828843.390925:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828843.390926:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390927:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828843.390930:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327284992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828843.390934:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828843.390935:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828843.390937:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828843.390938:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828843.390939:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828843.390940:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828843.390941:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828843.390942:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828843.390943:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828843.390944:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828843.390945:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:3.0:1787828843.390946:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828843.390947:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390948:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828843.390950:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828843.390951:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828843.390952:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828843.390953:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828843.900796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828843.900802:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828843.900804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828843.900809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828843.900814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828843.900818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828843.900819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828843.900822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828843.900824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828844.092726:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297664172, at 4297664192
00000100:00000010:3.0:1787828844.092746:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f300.
00000100:00000040:2.0:1787828844.092762:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297664172, at 4297664192
00000100:00000010:2.0:1787828844.092776:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fde0.
00000020:00000001:2.0:1787828844.220853:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828844.220862:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828844.220867:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828844.924757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828844.924762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828844.924764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828844.924767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828844.924771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828844.924773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828844.924775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828844.924777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828844.924780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828845.244853:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828845.244859:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828845.244862:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828845.948785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828845.948791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828845.948794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828845.948798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828845.948802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828845.948806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828845.948809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828845.948811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828845.948814:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828846.972778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828846.972785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828846.972787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828846.972791:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828846.972795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828846.972798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828846.972801:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828846.972803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828846.972805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828847.292813:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828847.292816:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828847.292819:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828847.996818:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828847.996826:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828847.996828:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828847.996833:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828847.996837:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828847.996841:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828847.996843:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828847.996846:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828847.996849:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828848.316882:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828848.316886:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828848.316889:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828848.508796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828848.508807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828848.508810:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828848.508814:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828848.508818:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828848.508827:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 2996
00000400:00000200:2.0:1787828848.508836:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828848.508842:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828848.508846:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828848.508848:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.508851:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.508853:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.508855:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828848.508859:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828848.508861:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828848.508864:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828848.508867:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828848.508869:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828848.508871:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828848.508873:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.508874:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.508875:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.508877:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828848.508879:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828848.508881:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828848.508883:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828848.508886:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828848.508890:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828848.508893:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828848.508895:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828848.508898:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828848.508911:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828848.508916:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828848.508920:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828848.508922:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828848.508926:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828848.508930:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828848.508935:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828848.508946:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828848.508953:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828848.508963:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828848.508979:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828848.508984:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828848.508988:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828848.509019:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828848.509023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828848.509028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828848.509030:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828848.509031:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828848.509035:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828848.509038:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2996
00000400:00000200:2.0:1787828848.509043:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828848.509046:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828848.509048:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828848.509050:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509053:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.509055:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509058:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828848.509061:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828848.509062:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828848.509064:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828848.509066:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828848.509069:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828848.509071:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828848.509074:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509077:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.509079:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509082:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828848.509085:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828848.509088:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509092:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828848.509094:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787828848.509099:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787828848.509102:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828848.509106:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509107:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828848.509109:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828848.509112:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:2.0:1787828848.509118:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828848.509124:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828848.509124:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828848.509127:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828848.509136:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000100:00000001:3.0:1787828848.509141:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787828848.509142:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:3.0:1787828848.509146:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828848.509148:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828848.509151:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.509153:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787828848.509154:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828848.509158:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828848.509159:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828848.509161:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.509165:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828848.509169:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00100000:3.0:1787828848.509173:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327285120:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787828848.509176:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509182:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:2.0:1787828848.509188:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787828848.509189:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828848.509194:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.509200:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787828848.509203:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828848.509206:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828848.509209:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
00000100:00000040:2.0:1787828848.509210:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787828848.509216:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828848.509226:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.509228:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787828848.509229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828848.509234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:3.0:1787828848.509234:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000100:00000001:2.0:1787828848.509236:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828848.509239:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828848.509242:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828848.509244:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2996
00000400:00000010:3.0:1787828848.509247:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000400:00000200:2.0:1787828848.509248:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828848.509254:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828848.509255:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828848.509258:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:3.0:1787828848.509260:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327285120, portal 25
00000100:00000001:2.0:1787828848.509261:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.509264:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828848.509267:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787828848.509268:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828848.509272:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787828848.509272:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000020:00000040:2.0:1787828848.509274:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828848.509277:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828848.509279:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828848.509282:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828848.509286:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828848.509287:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787828848.509290:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509295:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.509299:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509303:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828848.509303:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1787828848.509307:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000040:3.0:1787828848.509309:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787828848.509313:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000010:3.0:1787828848.509314:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000001:2.0:1787828848.509315:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828848.509319:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1787828848.509320:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327285120, offset 0
02000000:00000001:2.0:1787828848.509322:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828848.509326:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828848.509327:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787828848.509331:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000008:00100000:2.0:1787828848.509332:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828848.509335:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828848.509336:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828848.509340:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000400:00000200:3.0:1787828848.509344:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787828848.509345:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828848.509349:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828848.509350:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.509353:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787828848.509354:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828848.509357:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828848.509360:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.509361:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:2.0:1787828848.509364:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000400:00000200:3.0:1787828848.509368:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787828848.509370:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.509374:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00100000:2.0:1787828848.509378:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787828848.509380:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828848.509389:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [39999:39999:256:4294967295] 192.168.203.116@tcp LPNI seq info [39999:39999:8:4294967295]
00000400:00000200:3.0:1787828848.509402:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787828848.509408:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828848.509410:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828848.509418:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787828848.509421:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787828848.509435:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787828848.509435:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787828848.509440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828848.509443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000010:3.0:1787828848.509443:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000100:00000001:2.0:1787828848.509445:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1787828848.509466:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828848.509470:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828848.509475:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828848.509478:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828848.509481:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.509483:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828848.509492:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828848.509496:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828848.509502:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828848.509504:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828848.509509:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828848.509512:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.509521:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.509523:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.509529:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509530:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828848.509536:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.509539:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828848.509542:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297669292
00000100:00000010:3.0:1787828848.509546:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f300.
00000100:00000001:3.0:1787828848.509554:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828848.509561:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297669292
00000100:00000010:3.0:1787828848.509563:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f840.
00000100:00000001:3.0:1787828848.509565:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828848.509568:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828848.509570:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509573:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828848.509576:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848]  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828848.509583:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828848.509584:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828848.509586:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828848.509588:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828848.509589:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828848.509591:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828848.509593:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828848.509600:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787828848.509607:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828848.509607:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828848.509610:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828848.509612:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787828848.509614:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:2.0:1787828848.509617:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 2996
00000100:00000001:3.0:1787828848.509617:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828848.509622:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787828848.509624:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828848.509631:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828848.509636:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787828848.509637:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828848.509642:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:3.0:1787828848.509646:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828848.509647:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828848.509652:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509677:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828848.509679:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828848.509680:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828848.509681:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828848.509683:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828848.509688:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828848.509693:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:3.0:1787828848.509696:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1787828848.509697:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509701:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787828848.509701:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.509704:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1787828848.509707:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828848.509708:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828848.509710:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:3.0:1787828848.509712:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:2.0:1787828848.509716:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:3.0:1787828848.509716:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828848.509721:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787828848.509722:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828848.509727:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828848.509730:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00080000:3.0:1787828848.509731:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
02000000:00000001:2.0:1787828848.509735:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828848.509735:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:2.0:1787828848.509741:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828848.509743:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828848.509744:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828848.509748:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828848.509752:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828848.509756:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828848.509760:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327285120:192.168.203.116@tcp:250:kworker.0
00000100:00000010:2.0:1787828848.509762:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000100:00000001:3.0:1787828848.509764:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1787828848.509765:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:3.0:1787828848.509769:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787828848.509770:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828848.509773:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828848.509774:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327285120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828848.509776:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828848.509779:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828848.509782:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509788:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000010:2.0:1787828848.509789:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828848.509791:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:2.0:1787828848.509794:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828848.509795:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828848.509800:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00100000:2.0:1787828848.509803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000020:00000001:3.0:1787828848.509803:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828848.509806:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828848.509811:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828848.509815:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828848.509816:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828848.509820:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.509822:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000040:2.0:1787828848.509825:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787828848.509826:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828848.509829:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:3.0:1787828848.509835:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787828848.509837:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509838:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.509843:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787828848.509846:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3006)
00000100:00000001:3.0:1787828848.509851:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828848.509854:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.509856:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828848.509857:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828848.509858:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828848.509862:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.509868:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828848.509869:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.509871:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828848.509872:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828848.509874:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828848.509876:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327285248:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828848.509878:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828848.509880:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828848.509881:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.509882:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828848.509885:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828848.509886:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.509889:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828848.509891:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828848.509893:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828848.509896:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327285248, portal 10
00000100:00000001:3.0:1787828848.509897:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828848.509898:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:3.0:1787828848.509904:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.509909:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828848.509910:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828848.509912:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828848.509913:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327285248, offset 0
00000400:00000200:3.0:1787828848.509916:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828848.509919:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828848.509921:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828848.509923:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828848.509925:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828848.509928:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828848.509929:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828848.509932:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40000:40000:256:4294967295] 192.168.203.116@tcp LPNI seq info [40000:40000:8:4294967295]
00000400:00000200:3.0:1787828848.509935:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828848.509939:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828848.509942:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.509944:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828848.509946:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828848.509949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828848.509952:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828848.509970:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828848.509973:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.509974:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828848.509977:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828848.509979:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828848.509982:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828848.509983:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828848.509985:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828848.509988:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.509998:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.510000:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510004:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510005:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828848.510008:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828848.510011:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828848.510013:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828848.510014:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297668619
00000100:00000001:3.0:1787828848.510016:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297669292
00000100:00000001:3.0:1787828848.510018:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297668619
00000100:00000001:3.0:1787828848.510020:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297669292
00000100:00000001:3.0:1787828848.510021:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828848.510022:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510024:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828848.510026:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848]  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828848.510030:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828848.510031:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828848.510032:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828848.510034:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828848.510035:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828848.510036:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828848.510038:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828848.510044:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510047:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828848.510049:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510050:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828848.510051:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828848.510053:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510058:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828848.510060:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510064:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828848.510066:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510067:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828848.510067:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510069:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828848.510070:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828848.510072:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828848.510074:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828848.510075:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828848.510077:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828848.510082:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327285248:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828848.510084:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828848.510086:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510091:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828848.510092:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.510094:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828848.510095:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828848.510096:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828848.510099:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327285376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828848.510101:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828848.510102:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828848.510104:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.510105:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828848.510107:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828848.510108:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.510111:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828848.510112:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828848.510114:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828848.510116:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327285376, portal 4
00000100:00000001:3.0:1787828848.510118:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828848.510119:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:3.0:1787828848.510123:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510127:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828848.510128:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828848.510130:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828848.510131:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327285376, offset 0
00000400:00000200:3.0:1787828848.510134:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828848.510136:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828848.510138:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828848.510141:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828848.510142:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828848.510144:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828848.510146:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828848.510148:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40001:40001:256:4294967295] 192.168.203.116@tcp LPNI seq info [40001:40001:8:4294967295]
00000400:00000200:3.0:1787828848.510150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828848.510155:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828848.510158:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.510160:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828848.510162:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828848.510164:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828848.510167:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828848.510169:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828848.510172:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.510173:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828848.510175:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828848.510178:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828848.510179:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828848.510181:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828848.510182:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828848.510184:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510189:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.510191:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510195:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510196:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828848.510198:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828848.510199:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828848.510201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828848.510203:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297668626
00000100:00000001:3.0:1787828848.510204:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297669292
00000100:00000001:3.0:1787828848.510206:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297668626
00000100:00000001:3.0:1787828848.510208:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297669292
00000100:00000001:3.0:1787828848.510209:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828848.510210:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510211:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828848.510213:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848]  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828848.510218:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828848.510219:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828848.510220:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828848.510221:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828848.510222:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828848.510224:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828848.510226:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828848.510231:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510234:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828848.510235:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510236:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828848.510237:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828848.510239:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510243:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828848.510245:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510249:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828848.510251:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510252:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828848.510252:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510254:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828848.510255:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828848.510257:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828848.510259:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828848.510261:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828848.510263:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828848.510272:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327285376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828848.510275:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828848.510277:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510281:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828848.510281:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.510283:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828848.510284:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828848.510285:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828848.510287:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327285504:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828848.510288:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828848.510289:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828848.510290:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828848.510291:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828848.510293:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828848.510294:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828848.510296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828848.510297:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828848.510299:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828848.510301:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327285504, portal 4
00000100:00000001:3.0:1787828848.510303:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828848.510304:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:3.0:1787828848.510307:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510312:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828848.510313:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828848.510315:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828848.510316:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327285504, offset 0
00000400:00000200:3.0:1787828848.510318:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828848.510321:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828848.510323:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828848.510325:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828848.510327:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828848.510328:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828848.510330:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828848.510331:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40002:40002:256:4294967295] 192.168.203.116@tcp LPNI seq info [40002:40002:8:4294967295]
00000400:00000200:3.0:1787828848.510334:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828848.510338:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828848.510341:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.510343:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828848.510345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828848.510347:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787828848.510349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828848.510351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787828848.510354:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828848.510355:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f500.
00000400:00000200:3.0:1787828848.510357:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828848.510359:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828848.510361:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99ee0
00000400:00000010:3.0:1787828848.510362:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99ee0.
00000100:00000001:3.0:1787828848.510364:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828848.510366:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510371:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.510372:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510376:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510377:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828848.510379:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828848.510381:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828848.510383:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828848.510385:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297668631
00000100:00000001:3.0:1787828848.510386:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297669292
00000100:00000001:3.0:1787828848.510388:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297668631
00000100:00000001:3.0:1787828848.510389:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297669292
00000100:00000001:3.0:1787828848.510391:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828848.510392:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510393:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828848.510395:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848]  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828848.510399:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828848.510400:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828848.510401:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828848.510402:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828848.510403:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828848.510404:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828848.510407:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828848.510412:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510416:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828848.510416:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510418:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828848.510418:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828848.510420:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510424:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828848.510427:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510431:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828848.510433:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510434:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828848.510435:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510436:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828848.510437:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828848.510439:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828848.510441:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828848.510442:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828848.510444:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828848.510461:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327285504:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828848.510464:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510465:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.510467:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327285248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510471:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828848.510472:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828848.510473:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828848.510475:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828848.510476:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828848.510477:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828848.510479:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828848.510480:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:3.0:1787828848.510482:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828848.510483:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828848.510485:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:3.0:1787828848.510487:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828848.510488:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510489:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.510491:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327285376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510499:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828848.510500:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828848.510501:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828848.510503:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828848.510504:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828848.510505:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828848.510506:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828848.510507:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:3.0:1787828848.510508:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828848.510508:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828848.510509:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:3.0:1787828848.510510:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828848.510511:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510512:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828848.510514:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327285504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828848.510518:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828848.510518:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828848.510519:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3800.
02000000:00000001:3.0:1787828848.510520:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828848.510521:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828848.510522:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828848.510523:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828848.510524:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:3.0:1787828848.510525:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828848.510526:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828848.510526:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:3.0:1787828848.510528:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828848.510528:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510529:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828848.510532:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828848.510533:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828848.510534:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828848.510535:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828849.020799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828849.020806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828849.020808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828849.020813:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828849.020817:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828849.020821:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828849.020824:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828849.020826:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828849.020828:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828849.212795:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297669292, at 4297669312
00000100:00000040:3.0:1787828849.212795:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297669292, at 4297669312
00000100:00000010:3.0:1787828849.212814:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f840.
00000100:00000010:2.0:1787828849.212817:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f300.
00000100:00000001:2.0:1787828850.044779:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828850.044789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828850.044792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828850.044800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828850.044804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828850.044810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828850.044813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828850.044818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828850.044822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828850.364853:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828850.364857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828850.364860:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828851.068733:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828851.068743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828851.068746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828851.068753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828851.068758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828851.068763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828851.068767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828851.068771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828851.068775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828851.388835:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828851.388839:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828851.388842:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828852.092752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828852.092758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828852.092760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828852.092765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828852.092768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828852.092772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828852.092775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828852.092778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828852.092781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828853.116798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828853.116806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828853.116808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828853.116814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828853.116818:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828853.116822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828853.116825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828853.116827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828853.116830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828853.436794:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828853.436799:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828853.436802:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828853.628809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828853.628818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828853.628820:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828853.628823:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828853.628827:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828853.628833:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3001
00000400:00000200:3.0:1787828853.628840:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828853.628847:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828853.628850:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828853.628855:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.628858:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.628861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.628863:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828853.628868:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828853.628870:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828853.628873:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828853.628878:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828853.628881:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828853.628884:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828853.628888:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.628891:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.628895:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.628898:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828853.628902:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828853.628906:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828853.628910:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828853.628913:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828853.628916:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828853.628919:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.628922:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828853.628924:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828853.628932:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828853.628935:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828853.628938:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828853.628939:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.628942:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828853.628944:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828853.628948:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828853.628954:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787828853.628960:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828853.628968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327285632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828853.628976:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828853.629008:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828853.629013:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327285632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828853.629043:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828853.629047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828853.629051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828853.629053:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828853.629054:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828853.629058:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828853.629061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3001
00000400:00000200:3.0:1787828853.629067:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828853.629072:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828853.629075:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828853.629079:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629082:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.629086:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629090:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828853.629093:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828853.629095:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828853.629097:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828853.629099:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828853.629105:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828853.629110:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828853.629118:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629121:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.629128:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629130:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828853.629134:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828853.629140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629148:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828853.629154:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828853.629162:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828853.629168:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629172:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828853.629178:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828853.629187:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629192:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828853.629198:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828853.629205:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828853.629211:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828853.629228:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629233:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828853.629239:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828853.629244:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828853.629266:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828853.629272:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828853.629284:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327285760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828853.629302:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828853.629306:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828853.629311:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327285760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828853.629323:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828853.629329:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828853.629334:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828853.629338:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828853.629357:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828853.629359:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828853.629361:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3001
00000400:00000200:3.0:1787828853.629366:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828853.629372:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828853.629375:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828853.629379:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629383:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.629388:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629390:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828853.629394:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828853.629396:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828853.629400:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828853.629404:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828853.629407:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828853.629426:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828853.629432:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629435:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.629444:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629446:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828853.629447:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828853.629451:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629457:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828853.629460:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828853.629462:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828853.629468:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629471:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828853.629474:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828853.629478:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629481:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828853.629486:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828853.629491:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828853.629496:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828853.629497:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629513:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828853.629516:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828853.629519:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828853.629524:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828853.629529:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828853.629534:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327285888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828853.629549:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828853.629553:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828853.629558:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327285888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828853.629571:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828853.629574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828853.629579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828853.629580:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828853.629581:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828853.629587:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828853.629601:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3001
00000400:00000200:3.0:1787828853.629606:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828853.629617:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:0.1:1787828853.629619:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787828853.629620:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828853.629627:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.1:1787828853.629627:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:3.0:1787828853.629633:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.629637:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629641:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828853.629645:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828853.629648:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828853.629651:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828853.629653:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828853.629702:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828853.629707:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828853.629716:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629720:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828853.629724:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629726:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828853.629729:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828853.629735:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828853.629739:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828853.629741:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828853.629745:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828853.629750:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629754:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828853.629756:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828853.629762:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629766:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828853.629769:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828853.629773:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828853.629778:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828853.629779:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828853.629781:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828853.629783:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828853.629786:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828853.629790:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787828853.629796:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828853.629804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327286016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828853.629813:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828853.629815:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828853.629832:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327286016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828853.629839:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828853.629847:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3011)
00000800:00000200:0.1:1787828853.630197:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828853.630199:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828853.630431:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828853.630433:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828853.630676:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828853.630678:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787828854.140728:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828854.140735:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828854.140737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828854.140742:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828854.140745:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828854.140749:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828854.140751:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828854.140754:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828854.140756:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828854.332761:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297674413, at 4297674432
00000100:00000010:3.0:1787828854.332787:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000100:00000040:2.0:1787828854.332796:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297674413, at 4297674432
00000100:00000010:2.0:1787828854.332813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000020:00000001:2.0:1787828854.460926:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828854.460933:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828854.460938:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828855.164787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828855.164796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828855.164798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828855.164803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828855.164806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828855.164822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828855.164824:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828855.164827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828855.164830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828856.188760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828856.188768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828856.188770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828856.188776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828856.188780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828856.188784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828856.188787:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828856.188790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828856.188794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828856.508834:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828856.508837:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828856.508840:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828857.212816:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828857.212824:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828857.212828:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828857.212834:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828857.212839:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828857.212844:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828857.212848:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828857.212851:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828857.212855:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828857.532926:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828857.532931:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828857.532943:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828858.748821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828858.748838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828858.748842:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828858.748847:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828858.748853:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828858.748862:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3006
00000400:00000200:2.0:1787828858.748873:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828858.748882:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828858.748886:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828858.748889:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.748893:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.748896:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.748899:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828858.748919:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828858.748921:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828858.748924:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828858.748928:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828858.748930:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828858.748931:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828858.748934:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.748935:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.748937:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.748938:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828858.748941:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828858.748944:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828858.748946:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828858.748948:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828858.748951:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828858.748954:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.748956:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828858.748958:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828858.748986:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787828858.748990:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828858.748995:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828858.748996:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749001:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828858.749004:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828858.749010:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828858.749019:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828858.749027:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828858.749039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327286144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828858.749050:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828858.749055:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828858.749060:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327286144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828858.749107:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828858.749109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828858.749112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828858.749113:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828858.749114:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828858.749117:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828858.749120:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3006
00000400:00000200:2.0:1787828858.749124:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828858.749127:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828858.749129:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828858.749132:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749135:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.749142:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749145:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828858.749148:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828858.749149:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828858.749151:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828858.749153:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828858.749167:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828858.749170:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828858.749175:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749178:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.749180:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749181:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828858.749183:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828858.749186:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749193:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828858.749197:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828858.749204:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828858.749208:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749214:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828858.749221:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828858.749251:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749256:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828858.749266:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787828858.749274:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828858.749279:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828858.749283:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749286:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828858.749290:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828858.749295:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828858.749301:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828858.749310:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828858.749322:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327286272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828858.749335:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828858.749339:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828858.749346:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327286272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828858.749359:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828858.749362:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828858.749370:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828858.749390:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828858.749393:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828858.749399:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828858.749404:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3006
00000400:00000200:2.0:1787828858.749413:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828858.749423:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828858.749429:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828858.749439:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749445:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.749451:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749455:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828858.749463:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828858.749464:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828858.749466:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828858.749468:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828858.749474:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828858.749480:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828858.749484:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749486:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.749493:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749497:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828858.749500:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828858.749508:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749512:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828858.749515:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828858.749521:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828858.749525:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749529:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828858.749551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828858.749557:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749563:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828858.749571:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787828858.749576:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828858.749584:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828858.749588:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749593:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828858.749597:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828858.749617:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828858.749627:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828858.749635:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828858.749648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327286400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787828858.749648:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828858.749658:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828858.749660:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828858.749664:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327286400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828858.749696:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828858.749699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828858.749702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828858.749703:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828858.749705:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828858.749708:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1787828858.749721:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00080000:2.0:1787828858.749739:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3006
00000400:00000200:2.0:1787828858.749762:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828858.749766:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828858.749771:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828858.749783:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749788:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.749797:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749804:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828858.749812:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828858.749817:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828858.749823:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828858.749829:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828858.749836:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828858.749842:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828858.749855:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749862:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828858.749869:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749875:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828858.749878:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828858.749882:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828858.749887:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828858.749894:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828858.749901:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828858.749923:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749924:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828858.749927:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828858.749931:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749932:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828858.749936:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787828858.749940:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828858.749943:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828858.749944:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828858.749949:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828858.749950:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828858.749952:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828858.749958:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828858.749965:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828858.749994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327286528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828858.750016:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828858.750021:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828858.750030:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327286528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828858.750064:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828858.750074:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3016)
00000800:00000200:0.1:1787828858.750348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828858.750351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828858.750615:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828858.750617:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828858.750956:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828858.750958:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828859.452803:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297679533, at 4297679552
00000100:00000040:3.0:1787828859.452803:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297679533, at 4297679552
00000100:00000010:3.0:1787828859.452826:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000100:00000010:2.0:1787828859.452832:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000020:00000001:2.0:1787828859.580849:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828859.580853:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828859.580857:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828860.604849:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828860.604855:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828860.604861:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828863.676924:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828863.676929:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828863.676934:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828863.868809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828863.868825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828863.868829:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828863.868835:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828863.868842:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828863.868852:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3011
00000400:00000200:2.0:1787828863.868864:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828863.868874:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828863.868878:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828863.868883:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.868888:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.868892:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.868895:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828863.868901:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828863.868903:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828863.868907:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828863.868912:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828863.868915:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828863.868919:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828863.868935:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.868938:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.868942:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.868944:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828863.868948:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828863.868953:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828863.868957:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828863.868961:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828863.868967:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828863.868971:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.868974:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828863.868977:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828863.868991:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787828863.868996:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828863.869003:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828863.869005:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.869009:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828863.869013:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828863.869019:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828863.869030:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828863.869040:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828863.869054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327286656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828863.869069:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828863.869074:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828863.869081:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327286656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828863.869113:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828863.869118:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828863.869122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828863.869125:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828863.869127:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828863.869132:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828863.869138:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3011
00000400:00000200:2.0:1787828863.869144:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828863.869150:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828863.869159:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828863.869173:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869182:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.869191:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869202:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828863.869213:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828863.869220:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828863.869229:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828863.869240:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828863.869254:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828863.869263:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828863.869277:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869286:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.869299:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869310:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828863.869318:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828863.869329:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869361:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828863.869371:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828863.869381:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828863.869390:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.869396:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828863.869405:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828863.869416:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.869420:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828863.869429:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787828863.869437:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828863.869449:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828863.869453:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.869460:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828863.869465:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828863.869472:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828863.869488:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828863.869500:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828863.869519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327286784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787828863.869542:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828863.869552:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787828863.869566:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828863.869577:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828863.869590:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327286784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828863.869630:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828863.869638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828863.869651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828863.869697:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828863.869702:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828863.869712:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828863.869726:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3011
00000400:00000200:2.0:1787828863.869741:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828863.869757:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828863.869769:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828863.869781:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869787:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.869796:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869802:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828863.869811:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828863.869816:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828863.869821:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828863.869827:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828863.869839:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828863.869853:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828863.869867:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869877:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.869885:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869893:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828863.869903:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828863.869915:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828863.869920:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828863.869926:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828863.869933:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828863.869948:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.869959:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828863.869970:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828863.869978:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.869985:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828863.869999:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787828863.870008:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828863.870020:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828863.870025:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.870037:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828863.870043:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828863.870049:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828863.870060:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828863.870071:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828863.870086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327286912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828863.870119:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828863.870130:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828863.870144:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327286912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828863.870237:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828863.870254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828863.870267:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828863.870276:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828863.870281:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828863.870306:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828863.870316:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3011
00000400:00000200:2.0:1787828863.870334:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828863.870368:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828863.870376:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828863.870390:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.870402:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828863.870412:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.870424:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828863.870436:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828863.870441:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828863.870448:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828863.870455:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828863.870465:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828863.870477:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828863.870492:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.870521:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787828863.870523:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828863.870527:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787828863.870531:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828863.870542:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828863.870548:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828863.870555:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828863.870559:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828863.870562:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828863.870570:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828863.870581:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.870595:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828863.870603:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828863.870632:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.870643:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828863.870655:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787828863.870661:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828863.870665:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828863.870667:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828863.870670:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828863.870673:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828863.870678:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828863.870729:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828863.870739:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828863.870755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327287040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828863.870782:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828863.870789:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828863.870801:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327287040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828863.870841:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828863.870869:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3021)
00000800:00000200:0.1:1787828863.871072:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828863.871074:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828863.871265:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828863.871267:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:3.0:1787828864.572767:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297684653, at 4297684672
00000100:00000040:2.0:1787828864.572768:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297684653, at 4297684672
00000100:00000010:2.0:1787828864.572790:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000010:3.0:1787828864.572796:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000020:00000001:2.0:1787828866.748911:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828866.748917:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828866.748923:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828868.988798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828868.988810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828868.988812:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828868.988816:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828868.988821:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828868.988838:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3016
00000400:00000200:3.0:1787828868.988847:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828868.988854:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828868.988858:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828868.988861:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.988865:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.988867:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.988871:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828868.988875:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828868.988877:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828868.988880:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828868.988884:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828868.988887:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828868.988889:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828868.988893:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.988895:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.988897:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.988899:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828868.988903:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828868.988907:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828868.988910:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828868.988913:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828868.988917:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828868.988921:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.988923:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828868.988926:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828868.988935:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787828868.988940:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828868.988944:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828868.988946:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.988949:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828868.988952:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828868.988956:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828868.988963:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828868.988970:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828868.988980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327287168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828868.988992:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828868.988996:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828868.989003:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327287168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828868.989027:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828868.989030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828868.989034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828868.989035:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828868.989037:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828868.989041:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828868.989044:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3016
00000400:00000200:3.0:1787828868.989048:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828868.989052:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828868.989055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828868.989058:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989061:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.989064:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989066:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828868.989069:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828868.989071:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828868.989076:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828868.989079:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828868.989085:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828868.989090:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828868.989097:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989102:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.989106:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989112:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828868.989117:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828868.989124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989131:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828868.989136:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828868.989144:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828868.989149:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989153:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828868.989159:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828868.989166:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989171:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828868.989178:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787828868.989182:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828868.989188:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828868.989191:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989196:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828868.989200:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828868.989203:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828868.989209:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828868.989216:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828868.989224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327287296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828868.989243:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828868.989248:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828868.989257:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327287296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828868.989271:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828868.989275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828868.989283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828868.989287:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828868.989291:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828868.989297:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828868.989303:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3016
00000400:00000200:3.0:1787828868.989324:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828868.989333:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828868.989335:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828868.989342:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989345:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.989349:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989353:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828868.989358:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828868.989362:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828868.989365:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828868.989370:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828868.989375:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828868.989381:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828868.989391:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989397:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.989403:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989410:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828868.989415:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828868.989423:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989429:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828868.989435:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828868.989440:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828868.989446:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989449:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828868.989454:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828868.989460:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989466:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828868.989474:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000800:00000200:0.1:1787828868.989476:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787828868.989479:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.1:1787828868.989482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
02000000:00000001:3.0:1787828868.989484:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828868.989488:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989493:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828868.989496:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828868.989500:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828868.989507:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828868.989514:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828868.989524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327287424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828868.989538:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828868.989543:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828868.989554:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327287424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828868.989571:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828868.989577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828868.989583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828868.989587:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828868.989591:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828868.989595:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828868.989602:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3016
00000400:00000200:3.0:1787828868.989612:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828868.989629:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828868.989634:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828868.989642:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989648:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.989654:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989714:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828868.989720:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828868.989723:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828868.989727:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828868.989732:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828868.989738:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828868.989743:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828868.989752:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989758:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828868.989764:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989770:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828868.989774:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828868.989782:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828868.989789:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828868.989794:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828868.989799:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828868.989804:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989809:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828868.989814:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828868.989820:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989824:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828868.989831:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828868.989835:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828868.989838:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828868.989842:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828868.989847:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828868.989851:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828868.989855:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828868.989862:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828868.989868:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828868.989879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327287552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828868.989898:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828868.989903:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828868.989912:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327287552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828868.989932:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828868.989939:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3026)
00000800:00000200:0.1:1787828868.990156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828868.990158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828868.990482:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828868.990485:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787828868.990792:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828868.990795:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787828869.692727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297689772, at 4297689792
00000100:00000010:2.0:1787828869.692743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000100:00000040:3.0:1787828869.692745:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297689772, at 4297689792
00000100:00000010:3.0:1787828869.692755:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000020:00000001:2.0:1787828869.820860:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828869.820863:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828869.820866:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828870.524835:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828870.524854:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828870.524856:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828870.524863:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828870.524868:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828870.524872:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828870.524875:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828870.524878:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828870.524882:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828871.548750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828871.548758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828871.548760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828871.548765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828871.548769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828871.548773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828871.548775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828871.548779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828871.548781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787828872.892793:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828872.892797:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828872.892799:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828874.108758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828874.108766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828874.108768:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828874.108771:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828874.108775:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828874.108780:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3021
00000400:00000200:2.0:1787828874.108787:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828874.108792:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828874.108794:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828874.108797:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108799:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.108801:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108803:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828874.108812:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828874.108814:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828874.108816:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828874.108822:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828874.108823:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828874.108825:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828874.108828:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108829:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.108830:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108832:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828874.108834:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828874.108836:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108838:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828874.108840:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828874.108843:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828874.108844:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.108846:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828874.108847:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828874.108855:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787828874.108857:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828874.108860:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828874.108861:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.108863:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828874.108865:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828874.108869:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828874.108875:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828874.108880:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828874.108888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327287680/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828874.108895:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828874.108899:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828874.108902:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327287680/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828874.108917:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828874.108920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828874.108923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828874.108924:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828874.108925:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828874.108944:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828874.108946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3021
00000400:00000200:2.0:1787828874.108951:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828874.108953:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828874.108955:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828874.108957:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108959:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.108962:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108965:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828874.108968:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828874.108971:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828874.108972:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828874.108975:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828874.108978:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828874.108983:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828874.108990:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.108995:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.109000:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109005:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828874.109007:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828874.109011:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109016:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828874.109019:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828874.109024:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828874.109027:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109031:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828874.109052:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828874.109057:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109060:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828874.109065:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828874.109067:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828874.109071:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828874.109074:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109076:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828874.109077:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828874.109079:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828874.109083:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828874.109089:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828874.109097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327287808/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828874.109107:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828874.109110:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828874.109114:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327287808/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828874.109127:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828874.109130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828874.109132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828874.109135:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828874.109138:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828874.109144:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828874.109148:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3021
00000400:00000200:2.0:1787828874.109153:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828874.109160:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828874.109163:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828874.109167:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109173:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.109175:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109179:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828874.109183:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828874.109184:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828874.109185:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828874.109186:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828874.109188:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828874.109191:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828874.109196:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109198:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.109201:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109207:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828874.109208:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828874.109213:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109216:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828874.109218:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828874.109222:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828874.109226:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109230:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828874.109233:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828874.109236:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109241:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828874.109248:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828874.109251:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828874.109256:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828874.109259:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109263:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828874.109266:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828874.109283:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828874.109287:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787828874.109289:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828874.109293:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327287936/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828874.109303:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828874.109304:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828874.109306:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327287936/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828874.109311:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828874.109313:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828874.109315:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828874.109316:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828874.109317:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828874.109318:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828874.109320:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3021
00000400:00000200:2.0:1787828874.109323:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828874.109325:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828874.109327:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828874.109328:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109330:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.109331:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109332:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828874.109334:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828874.109334:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828874.109335:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828874.109336:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828874.109338:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828874.109339:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828874.109341:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109342:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828874.109343:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109345:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828874.109347:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828874.109348:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828874.109350:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828874.109351:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828874.109353:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828874.109354:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109355:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828874.109357:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828874.109358:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109359:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828874.109361:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828874.109362:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828874.109364:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828874.109365:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828874.109366:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828874.109367:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828874.109368:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828874.109370:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787828874.109373:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828874.109377:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327288064/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828874.109385:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828874.109387:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828874.109391:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327288064/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828874.109398:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828874.109403:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3031)
00000800:00000200:0.1:1787828874.109441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828874.109454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828874.109744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828874.109745:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828874.109985:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828874.109987:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828874.110214:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828874.110216:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787828874.812743:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297694892, at 4297694912
00000100:00000040:3.0:1787828874.812743:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297694892, at 4297694912
00000100:00000010:2.0:1787828874.812757:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000100:00000010:3.0:1787828874.812760:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000020:00000001:2.0:1787828875.964915:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828875.964919:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828875.964922:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787828879.036888:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787828879.036894:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787828879.036898:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787828879.228779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828879.228793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828879.228796:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828879.228809:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828879.228814:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828879.228827:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3026
00000400:00000200:2.0:1787828879.228854:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828879.228861:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828879.228864:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828879.228866:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.228869:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.228872:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.228875:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828879.228879:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828879.228881:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828879.228883:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828879.228887:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828879.228890:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828879.228892:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828879.228896:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.228898:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.228901:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.228903:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828879.228907:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828879.228910:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828879.228913:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828879.228916:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828879.228921:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828879.228923:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.228925:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828879.228927:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828879.228935:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828879.228938:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828879.228941:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828879.228942:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.228945:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828879.228947:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828879.228950:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828879.228956:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787828879.228961:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828879.228968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327288192/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828879.228976:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828879.228981:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828879.228986:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327288192/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828879.229010:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828879.229014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828879.229018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828879.229020:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828879.229021:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828879.229025:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828879.229029:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3026
00000400:00000200:2.0:1787828879.229033:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828879.229038:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828879.229041:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828879.229044:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229045:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.229047:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229049:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828879.229052:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828879.229054:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828879.229060:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828879.229062:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828879.229067:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828879.229070:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828879.229074:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229077:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.229080:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229083:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828879.229087:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828879.229095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229104:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828879.229113:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828879.229123:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828879.229137:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229158:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828879.229164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828879.229170:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229180:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828879.229190:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828879.229200:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828879.229211:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828879.229219:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229227:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828879.229228:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828879.229231:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828879.229252:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787828879.229258:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828879.229291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327288320/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828879.229307:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828879.229317:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828879.229328:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327288320/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828879.229370:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828879.229373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828879.229383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828879.229393:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828879.229402:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828879.229407:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828879.229411:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3026
00000400:00000200:2.0:1787828879.229424:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828879.229429:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828879.229431:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828879.229438:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229441:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.229444:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229451:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828879.229460:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828879.229463:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828879.229465:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828879.229472:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828879.229479:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828879.229489:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828879.229496:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229507:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.229517:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229525:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828879.229534:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828879.229544:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229549:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828879.229552:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828879.229561:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828879.229573:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229584:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828879.229591:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828879.229594:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229596:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828879.229603:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828879.229607:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828879.229613:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828879.229617:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229626:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828879.229628:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828879.229634:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828879.229639:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000800:00000200:0.1:1787828879.229643:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828879.229644:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828879.229660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327288448/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828879.229666:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828879.229667:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828879.229678:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000100:00000040:2.0:1787828879.229700:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327288448/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828879.229708:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828879.229712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828879.229717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828879.229723:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828879.229727:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828879.229734:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828879.229740:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3026
00000400:00000200:2.0:1787828879.229746:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828879.229752:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828879.229755:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828879.229762:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229773:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.229783:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229792:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828879.229800:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828879.229804:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828879.229808:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828879.229810:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828879.229816:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828879.229824:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828879.229835:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229839:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828879.229845:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229849:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828879.229857:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828879.229865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828879.229871:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828879.229877:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828879.229883:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828879.229893:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229903:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828879.229912:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828879.229918:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229921:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828879.229929:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828879.229935:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828879.229943:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828879.229946:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828879.229955:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828879.229964:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828879.229969:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828879.229997:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787828879.230011:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828879.230028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327288576/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828879.230123:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828879.230129:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828879.230146:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327288576/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828879.230197:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828879.230209:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3037)
00000800:00000200:0.1:1787828879.230418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828879.230419:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787828879.230684:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828879.230687:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787828879.230914:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828879.230916:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000100:00000040:2.0:1787828879.932790:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297700013, at 4297700032
00000100:00000040:3.0:1787828879.932790:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297700013, at 4297700032
00000100:00000010:2.0:1787828879.932808:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000100:00000010:3.0:1787828879.932811:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000100:00080000:3.0:1787828884.348756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828884.348765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828884.348767:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828884.348770:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828884.348774:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828884.348780:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3032
00000400:00000200:3.0:1787828884.348787:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828884.348792:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828884.348795:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828884.348797:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348800:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.348802:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348804:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828884.348807:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828884.348808:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828884.348810:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828884.348813:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828884.348815:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828884.348817:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828884.348820:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348821:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.348823:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348824:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828884.348826:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828884.348828:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348830:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828884.348832:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828884.348834:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828884.348837:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.348838:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828884.348840:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828884.348847:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828884.348849:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828884.348853:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828884.348854:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.348856:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828884.348858:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828884.348862:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828884.348868:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787828884.348873:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828884.348880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327288704/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828884.348888:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828884.348890:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828884.348894:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327288704/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828884.348912:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828884.348914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828884.348917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828884.348918:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828884.348919:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828884.348921:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828884.348923:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3032
00000400:00000200:3.0:1787828884.348925:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828884.348927:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828884.348928:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828884.348931:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348933:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.348935:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348937:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828884.348939:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828884.348940:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828884.348941:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828884.348942:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828884.348945:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828884.348949:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828884.348952:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348956:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.348958:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348962:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828884.348964:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828884.348968:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828884.348970:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828884.348974:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828884.348978:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828884.348982:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.348986:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828884.348990:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828884.348993:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.348998:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828884.349001:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828884.349005:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828884.349007:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828884.349009:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349013:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828884.349014:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828884.349017:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828884.349023:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828884.349028:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828884.349035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327288832/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828884.349044:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828884.349047:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828884.349052:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327288832/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828884.349060:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828884.349063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828884.349067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828884.349068:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828884.349071:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828884.349073:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828884.349078:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3032
00000400:00000200:3.0:1787828884.349081:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828884.349086:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828884.349088:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828884.349090:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349095:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.349097:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349099:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828884.349101:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828884.349104:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828884.349107:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828884.349109:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828884.349113:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828884.349118:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828884.349122:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349125:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.349128:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349132:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828884.349137:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828884.349142:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349146:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828884.349150:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828884.349153:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828884.349158:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349162:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828884.349164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828884.349170:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349172:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828884.349175:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828884.349178:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828884.349180:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828884.349183:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349185:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828884.349188:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828884.349190:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828884.349193:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828884.349195:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828884.349218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327288960/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828884.349228:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828884.349230:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828884.349235:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327288960/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828884.349260:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828884.349264:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828884.349268:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828884.349273:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828884.349277:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828884.349283:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828884.349290:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3032
00000400:00000200:3.0:1787828884.349296:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828884.349302:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828884.349305:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828884.349310:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349315:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.349317:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349319:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828884.349321:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828884.349324:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828884.349327:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828884.349330:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828884.349335:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828884.349341:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828884.349347:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349352:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828884.349356:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349360:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828884.349362:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828884.349365:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828884.349387:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828884.349388:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828884.349390:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828884.349395:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349396:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828884.349400:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828884.349403:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349405:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828884.349414:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:3.0:1787828884.349415:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828884.349420:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000800:00000200:0.1:1787828884.349421:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
02000000:00000001:3.0:1787828884.349422:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828884.349425:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828884.349428:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828884.349429:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828884.349433:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828884.349455:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828884.349461:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828884.349467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327289088/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828884.349485:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828884.349489:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828884.349500:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327289088/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828884.349519:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828884.349528:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3042)
00000800:00000200:0.1:1787828884.349981:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828884.349984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787828884.350346:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828884.350349:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787828884.350782:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828884.350786:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000040:2.0:1787828885.052756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297705132, at 4297705152
00000100:00000010:2.0:1787828885.052773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000100:00000040:3.0:1787828885.052787:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297705132, at 4297705152
00000100:00000010:3.0:1787828885.052798:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000100:00000001:3.0:1787828889.469084:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828889.469092:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828889.469097:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828889.469109:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.469126:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828889.469130:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.469136:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828889.469140:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828889.469143:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828889.469148:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327289216:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828889.469155:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828889.469160:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828889.469176:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.469180:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828889.469187:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828889.469192:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828889.469213:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828889.469219:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98e18.
00000400:00000010:3.0:1787828889.469232:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f993b8.
00000100:00000200:3.0:1787828889.469243:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289216, portal 25
00000100:00000001:3.0:1787828889.469249:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828889.469254:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:3.0:1787828889.469268:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.469292:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828889.469298:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828889.469304:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99ee0.
00000100:00000200:3.0:1787828889.469309:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327289216, offset 0
00000400:00000200:3.0:1787828889.469321:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828889.469332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.469341:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828889.469350:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828889.469356:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828889.469363:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828889.469369:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828889.469377:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40031:40031:256:4294967295] 192.168.203.116@tcp LPNI seq info [40031:40031:8:4294967295]
00000400:00000200:3.0:1787828889.469388:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.469404:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828889.469413:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828889.469420:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f500.
00000800:00000200:3.0:1787828889.469432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828889.469441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828889.469462:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828889.469471:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828889.469477:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297710252
00000100:00000010:3.0:1787828889.469485:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fd80.
00000100:00000001:3.0:1787828889.469494:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828889.469503:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297710252
00000100:00000010:3.0:1787828889.469511:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f720.
00000100:00000001:3.0:1787828889.469516:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828889.469522:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828889.469527:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.469532:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.469535:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828889.469545:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828889.469549:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828889.469554:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828889.469556:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828889.469559:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828889.469566:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.469576:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828889.469579:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.469582:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828889.469584:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828889.469585:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828889.469588:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289344:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828889.469591:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828889.469592:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828889.469594:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.469595:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828889.469599:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828889.469601:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828889.469605:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828889.469609:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f989a0.
00000400:00000010:3.0:1787828889.469613:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828889.469617:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289344, portal 10
00000100:00000001:3.0:1787828889.469619:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828889.469620:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:3.0:1787828889.469629:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.469644:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828889.469647:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828889.469651:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99a18.
00000100:00000200:3.0:1787828889.469654:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327289344, offset 0
00000400:00000200:3.0:1787828889.469683:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828889.469686:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.469691:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828889.469693:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828889.469696:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828889.469700:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828889.469703:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828889.469707:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40032:40032:255:4294967295] 192.168.203.116@tcp LPNI seq info [40032:40032:7:4294967295]
00000400:00000200:3.0:1787828889.469712:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.469721:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828889.469726:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828889.469731:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f900.
00000800:00000200:3.0:1787828889.469753:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828889.469760:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828889.469765:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828889.469771:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828889.469776:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828889.469778:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297709579
00000100:00000001:3.0:1787828889.469782:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297710252
00000100:00000001:3.0:1787828889.469785:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297709579
00000100:00000001:3.0:1787828889.469788:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297710252
00000100:00000001:3.0:1787828889.469791:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828889.469794:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.469797:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828889.469803:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.469836:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828889.469838:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.469840:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828889.469842:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828889.469845:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828889.469848:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289472:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828889.469851:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828889.469853:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828889.469855:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.469857:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828889.469860:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828889.469863:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828889.469866:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828889.469869:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828889.469873:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828889.469876:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289472, portal 4
00000100:00000001:3.0:1787828889.469878:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828889.469879:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:3.0:1787828889.469902:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.469912:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828889.469915:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828889.469918:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99990.
00000100:00000200:3.0:1787828889.469921:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327289472, offset 0
00000400:00000200:3.0:1787828889.469928:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828889.469933:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.469938:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828889.469942:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828889.469946:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828889.469949:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828889.469967:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828889.469970:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40033:40033:254:4294967295] 192.168.203.116@tcp LPNI seq info [40033:40033:6:4294967295]
00000400:00000200:3.0:1787828889.469974:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.469981:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828889.469986:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828889.469989:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f600.
00000800:00000200:3.0:1787828889.469993:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828889.469997:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828889.470000:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828889.470003:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828889.470006:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828889.470008:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297709586
00000100:00000001:3.0:1787828889.470011:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297710252
00000100:00000001:3.0:1787828889.470013:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297709586
00000100:00000001:3.0:1787828889.470016:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297710252
00000100:00000001:3.0:1787828889.470018:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828889.470020:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.470022:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.470024:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828889.470028:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828889.470029:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828889.470032:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828889.470033:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828889.470034:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828889.470038:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.470045:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828889.470046:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.470048:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828889.470050:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828889.470052:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828889.470055:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289600:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828889.470061:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828889.470062:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828889.470064:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828889.470066:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828889.470069:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828889.470071:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828889.470075:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828889.470077:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98dc0.
00000400:00000010:3.0:1787828889.470081:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99198.
00000100:00000200:3.0:1787828889.470084:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289600, portal 4
00000100:00000001:3.0:1787828889.470086:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828889.470087:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:3.0:1787828889.470093:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828889.470100:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828889.470103:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828889.470106:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99660.
00000100:00000200:3.0:1787828889.470108:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327289600, offset 0
00000400:00000200:3.0:1787828889.470112:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828889.470116:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.470119:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828889.470122:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828889.470125:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828889.470127:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828889.470130:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828889.470132:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40034:40034:253:4294967295] 192.168.203.116@tcp LPNI seq info [40034:40034:5:4294967295]
00000400:00000200:3.0:1787828889.470137:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828889.470143:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828889.470147:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828889.470151:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f700.
00000800:00000200:3.0:1787828889.470155:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828889.470160:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828889.470178:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828889.470181:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828889.470185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828889.470187:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297709591
00000100:00000001:3.0:1787828889.470190:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297710252
00000100:00000001:3.0:1787828889.470193:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297709591
00000100:00000001:3.0:1787828889.470196:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297710252
00000100:00000001:3.0:1787828889.470198:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828889.470200:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.470203:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.470205:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828889.470208:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828889.470209:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787828889.470211:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828889.470213:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828889.470214:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828889.470216:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828890.172727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297710252, at 4297710272
00000100:00000040:3.0:1787828890.172727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297710252, at 4297710272
00000100:00000010:2.0:1787828890.172745:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fd80.
00000100:00000010:3.0:1787828890.172746:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f720.
00000100:00000001:2.0:1787828892.028841:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828892.028852:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828892.028855:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828892.028863:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828892.028868:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828892.028873:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828892.028877:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828892.028880:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828892.028884:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828892.541456:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828892.541464:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828892.541471:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828892.541485:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828892]  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828892.541517:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828892.541523:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828892.541530:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98e18.
00000400:00000200:3.0:1787828892.541538:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f993b8
00000400:00000010:3.0:1787828892.541543:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f993b8.
00000100:00000001:3.0:1787828892.541551:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828892.541561:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828892.541591:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.541627:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828892.541633:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.541639:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.541643:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828892.541654:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.541697:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828892.541705:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.541714:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828892.541718:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.541720:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.541722:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.541725:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828892.541729:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828892.541737:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828892.541748:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828892.541845:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828892.541855:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828892.541866:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327289216:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828892.541872:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828892.541875:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828892.541879:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327289216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.541893:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828892.541898:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828892.541905:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828892.541910:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828892.541915:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828892.541929:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828892.541933:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828892.541937:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:3.0:1787828892.541942:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828892.541945:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828892.541947:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:3.0:1787828892.541951:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828892.541953:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.541956:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828892.541958:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828892.541960:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828892.541961:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828892.541965:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828892]  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828892.541979:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828892.541981:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828892.541984:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98dc0.
00000400:00000200:3.0:1787828892.541987:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99198
00000400:00000010:3.0:1787828892.541989:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99198.
00000100:00000001:3.0:1787828892.541992:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828892.541995:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828892.542004:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542011:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828892.542013:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542016:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.542017:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828892.542022:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542030:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828892.542034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542041:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828892.542044:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542046:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.542048:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542050:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828892.542052:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828892.542055:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828892.542060:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828892.542063:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828892.542067:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828892.542076:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289600:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828892.542081:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828892.542085:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828892]  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828892.542099:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828892.542100:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828892.542103:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f989a0.
00000400:00000200:3.0:1787828892.542106:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828892.542108:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828892.542111:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828892.542114:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828892.542128:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542134:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828892.542135:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542137:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.542138:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828892.542158:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542166:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828892.542170:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542177:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828892.542181:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542183:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.542184:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542186:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828892.542188:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828892.542191:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828892.542195:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828892.542198:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828892.542202:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828892.542211:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289344:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828892.542215:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828892.542218:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828892]  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828892.542229:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828892.542230:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828892.542232:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828892.542234:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828892.542236:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828892.542239:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828892.542242:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828892.542251:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542258:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828892.542259:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542262:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.542263:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828892.542267:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542274:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828892.542284:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542292:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828892.542295:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542297:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828892.542298:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542300:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828892.542302:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828892.542305:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828892.542309:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828892.542312:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828892.542317:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828892.542325:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289472:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828892.542329:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542332:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828892.542352:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327289600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542358:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828892.542360:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828892.542362:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828892.542366:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828892.542368:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828892.542369:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828892.542372:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828892.542374:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:3.0:1787828892.542376:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828892.542378:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828892.542379:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:3.0:1787828892.542382:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828892.542384:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542386:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828892.542390:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327289344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542397:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828892.542413:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828892.542414:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828892.542417:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828892.542418:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828892.542420:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828892.542422:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828892.542423:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:3.0:1787828892.542425:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828892.542427:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828892.542429:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:3.0:1787828892.542431:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828892.542432:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542434:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828892.542437:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327289472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828892.542443:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828892.542445:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828892.542446:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828892.542448:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828892.542450:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828892.542451:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828892.542453:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828892.542454:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:3.0:1787828892.542456:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828892.542458:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828892.542459:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:3.0:1787828892.542462:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828892.542463:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542465:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828892.542475:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828892.542477:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828892.542479:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828892.542481:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828893.052805:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828893.052814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828893.052817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828893.052823:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828893.052829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828893.052836:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828893.052841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828893.052846:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828893.052851:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828894.588785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828894.588803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828894.588807:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828894.588814:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828894.588835:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828894.588845:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3042
00000400:00000200:2.0:1787828894.588857:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828894.588865:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828894.588870:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828894.588874:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.588878:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.588881:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.588885:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828894.588890:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828894.588893:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828894.588896:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828894.588901:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828894.588920:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828894.588923:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828894.588926:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.588929:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.588932:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.588934:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828894.588938:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828894.588942:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828894.588946:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828894.588950:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828894.588955:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828894.588959:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.588962:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828894.588965:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828894.588973:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787828894.588978:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828894.588983:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828894.588985:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.588989:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828894.588992:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828894.588998:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828894.589020:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828894.589033:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828894.589066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828894.589079:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828894.589084:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828894.589090:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828894.589117:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828894.589150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828894.589154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828894.589156:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828894.589158:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828894.589162:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828894.589165:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3042
00000400:00000200:2.0:1787828894.589169:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828894.589173:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828894.589187:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828894.589190:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589193:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.589196:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589198:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828894.589201:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828894.589202:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828894.589204:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828894.589207:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828894.589209:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828894.589212:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828894.589215:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589217:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.589219:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589221:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828894.589224:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828894.589228:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589231:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828894.589233:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828894.589240:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828894.589242:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589244:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828894.589251:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828894.589254:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589257:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828894.589263:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787828894.589266:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828894.589269:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828894.589270:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589272:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828894.589274:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828894.589276:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828894.589281:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828894.589285:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828894.589291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828894.589300:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828894.589303:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828894.589307:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828894.589317:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828894.589320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828894.589324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828894.589326:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828894.589327:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828894.589330:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828894.589333:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3042
00000400:00000200:2.0:1787828894.589338:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828894.589341:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828894.589343:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828894.589346:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589349:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.589351:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589354:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828894.589356:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828894.589358:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828894.589360:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828894.589362:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828894.589365:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828894.589368:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828894.589371:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589373:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.589376:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589378:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828894.589381:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828894.589384:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589387:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828894.589389:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828894.589393:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828894.589395:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589398:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828894.589401:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828894.589404:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589406:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828894.589410:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787828894.589413:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828894.589416:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828894.589417:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589419:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828894.589421:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828894.589423:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828894.589427:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828894.589437:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828894.589443:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828894.589452:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828894.589454:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828894.589458:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828894.589467:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828894.589470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828894.589474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828894.589476:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828894.589478:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828894.589480:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828894.589484:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3042
00000400:00000200:2.0:1787828894.589488:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828894.589492:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828894.589494:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828894.589498:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589500:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.589502:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589505:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828894.589507:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828894.589509:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828894.589510:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828894.589513:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828894.589515:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828894.589517:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828894.589521:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589523:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828894.589525:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589528:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828894.589530:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828894.589533:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828894.589536:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828894.589539:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828894.589541:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828894.589543:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589546:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828894.589548:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828894.589551:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589552:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828894.589557:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787828894.589559:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828894.589562:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828894.589565:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828894.589567:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828894.589568:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828894.589570:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828894.589575:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828894.589579:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828894.589584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828894.589592:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828894.589594:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828894.589598:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828894.589607:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828894.589612:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3052)
00000100:00000001:3.0:1787828894.589795:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828894.589806:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828894.589810:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828894.589826:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.589842:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828894.589845:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.589850:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828894.589854:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828894.589858:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828894.589862:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327289728:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828894.589869:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828894.589874:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828894.589877:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.589881:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828894.589892:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828894.589894:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828894.589903:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828894.589908:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828894.589917:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828894.589926:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289728, portal 25
00000100:00000001:3.0:1787828894.589929:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828894.589931:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402436160 : -116696307115456 : ffff95dd86115a40)
00000100:00000040:3.0:1787828894.589938:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.589947:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828894.589951:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828894.589954:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828894.589957:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327289728, offset 0
00000400:00000200:3.0:1787828894.589964:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828894.589975:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.589984:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828894.589990:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828894.589995:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828894.589999:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828894.590001:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828894.590007:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40035:40035:256:4294967295] 192.168.203.116@tcp LPNI seq info [40035:40035:8:4294967295]
00000400:00000200:3.0:1787828894.590011:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.590022:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828894.590030:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.590036:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828894.590043:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828894.590048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828894.590055:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828894.590059:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828894.590064:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.590067:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828894.590074:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828894.590079:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828894.590085:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828894.590088:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828894.590092:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828894.590097:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590110:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.590114:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590122:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.590124:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828894.590146:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828894.590152:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828894.590155:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297715373
00000100:00000010:3.0:1787828894.590162:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f4e0.
00000100:00000001:3.0:1787828894.590171:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828894.590179:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297715373
00000100:00000010:3.0:1787828894.590191:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f900.
00000100:00000001:3.0:1787828894.590195:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828894.590199:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828894.590203:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.590206:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828894.590212:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894]  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828894.590225:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828894.590227:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828894.590229:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828894.590232:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828894.590234:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828894.590238:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828894.590242:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828894.590253:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590261:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828894.590264:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.590267:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.590270:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828894.590275:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590823:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828894.590831:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590841:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828894.590844:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.590847:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.590848:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.590851:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828894.590854:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828894.590859:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828894.590864:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828894.590867:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828894.590872:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828894.590882:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327289728:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828894.590891:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828894.590896:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590905:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828894.590906:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.590910:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828894.590913:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828894.590915:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828894.590919:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289856:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828894.590923:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828894.590925:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828894.590928:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.590930:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828894.590936:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828894.590938:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828894.590943:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828894.590947:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828894.590952:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828894.590956:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289856, portal 10
00000100:00000001:3.0:1787828894.590959:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828894.590960:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:3.0:1787828894.590968:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.590976:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828894.590980:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828894.590982:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828894.590986:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327289856, offset 0
00000400:00000200:3.0:1787828894.590991:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828894.590996:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.591001:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828894.591021:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828894.591025:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828894.591029:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828894.591032:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828894.591036:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40036:40036:256:4294967295] 192.168.203.116@tcp LPNI seq info [40036:40036:8:4294967295]
00000400:00000200:3.0:1787828894.591041:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.591050:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828894.591057:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.591061:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828894.591066:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828894.591070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828894.591077:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828894.591081:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828894.591086:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.591089:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828894.591094:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828894.591098:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828894.591102:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828894.591105:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828894.591108:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828894.591113:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591124:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.591143:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591169:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.591172:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828894.591177:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828894.591181:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828894.591185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828894.591187:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297714700
00000100:00000001:3.0:1787828894.591191:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297715373
00000100:00000001:3.0:1787828894.591194:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297714700
00000100:00000001:3.0:1787828894.591197:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297715373
00000100:00000001:3.0:1787828894.591200:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828894.591203:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.591205:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828894.591211:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894]  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828894.591220:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828894.591222:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828894.591224:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828894.591227:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828894.591229:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828894.591232:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828894.591236:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828894.591246:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591253:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828894.591255:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591258:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.591260:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828894.591264:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591272:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828894.591277:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591296:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828894.591303:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591305:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.591306:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591309:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828894.591312:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828894.591315:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828894.591320:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828894.591323:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828894.591329:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828894.591354:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289856:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828894.591360:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828894.591365:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591374:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828894.591376:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.591379:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828894.591381:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828894.591384:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828894.591388:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289984:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828894.591392:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828894.591395:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828894.591412:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.591414:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828894.591418:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828894.591421:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828894.591426:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828894.591429:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828894.591434:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828894.591438:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327289984, portal 4
00000100:00000001:3.0:1787828894.591440:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828894.591442:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:3.0:1787828894.591449:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591457:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828894.591460:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828894.591463:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828894.591466:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327289984, offset 0
00000400:00000200:3.0:1787828894.591471:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828894.591476:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.591481:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828894.591485:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828894.591488:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828894.591491:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828894.591494:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828894.591498:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40037:40037:256:4294967295] 192.168.203.116@tcp LPNI seq info [40037:40037:8:4294967295]
00000400:00000200:3.0:1787828894.591503:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.591510:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828894.591516:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.591519:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828894.591524:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828894.591527:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828894.591532:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828894.591535:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828894.591539:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.591542:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828894.591545:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828894.591549:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828894.591553:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828894.591555:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828894.591558:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828894.591562:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591571:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.591576:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591590:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.591592:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828894.591595:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828894.591599:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828894.591604:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828894.591606:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297714707
00000100:00000001:3.0:1787828894.591609:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297715373
00000100:00000001:3.0:1787828894.591612:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297714707
00000100:00000001:3.0:1787828894.591614:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297715373
00000100:00000001:3.0:1787828894.591617:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828894.591619:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.591622:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828894.591626:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894]  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828894.591635:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828894.591637:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828894.591640:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828894.591642:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828894.591644:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828894.591647:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828894.591651:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828894.591661:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591668:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828894.591669:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591671:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.591673:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828894.591678:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591709:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828894.591715:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591724:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828894.591726:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591729:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.591730:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591732:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828894.591734:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828894.591736:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828894.591739:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828894.591741:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828894.591744:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828894.591750:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327289984:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828894.591753:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828894.591756:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591760:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828894.591762:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.591763:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828894.591765:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828894.591767:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828894.591769:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327290112:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828894.591771:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828894.591773:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828894.591775:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828894.591776:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828894.591779:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828894.591781:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828894.591783:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828894.591785:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828894.591788:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828894.591791:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327290112, portal 4
00000100:00000001:3.0:1787828894.591792:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828894.591793:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:3.0:1787828894.591798:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591803:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828894.591805:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828894.591806:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828894.591808:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327290112, offset 0
00000400:00000200:3.0:1787828894.591811:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828894.591814:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.591817:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828894.591819:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828894.591821:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828894.591823:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828894.591825:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828894.591827:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40038:40038:256:4294967295] 192.168.203.116@tcp LPNI seq info [40038:40038:8:4294967295]
00000400:00000200:3.0:1787828894.591830:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828894.591835:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828894.591839:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.591841:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828894.591844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828894.591846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787828894.591850:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828894.591852:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787828894.591854:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828894.591857:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828894.591859:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828894.591861:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828894.591864:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828894.591865:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828894.591867:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828894.591869:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591877:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.591881:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591889:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.591890:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828894.591894:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828894.591897:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828894.591901:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828894.591903:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297714713
00000100:00000001:3.0:1787828894.591906:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297715373
00000100:00000001:3.0:1787828894.591908:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297714713
00000100:00000001:3.0:1787828894.591911:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297715373
00000100:00000001:3.0:1787828894.591914:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828894.591916:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.591919:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828894.591924:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894]  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828894.591933:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828894.591935:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828894.591936:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828894.591938:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828894.591940:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828894.591941:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828894.591944:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828894.591950:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591955:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828894.591956:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591957:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.591958:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828894.591961:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591966:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828894.591969:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.591973:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828894.591975:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591976:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828894.591977:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591978:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828894.591979:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828894.591981:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828894.591983:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828894.591985:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828894.591987:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828894.591993:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327290112:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828894.591995:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.591997:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.592000:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115a40 x1874671327289728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.592005:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828894.592007:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828894.592009:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:3.0:1787828894.592011:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828894.592014:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828894.592015:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828894.592017:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828894.592019:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:3.0:1787828894.592022:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828894.592024:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828894.592025:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115a40.
00000100:00000001:3.0:1787828894.592028:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828894.592029:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.592030:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.592032:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327289856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.592037:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828894.592038:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828894.592039:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3400.
02000000:00000001:3.0:1787828894.592041:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828894.592041:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828894.592042:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828894.592043:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828894.592044:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:3.0:1787828894.592046:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828894.592047:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828894.592047:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:3.0:1787828894.592050:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828894.592051:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.592052:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.592054:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327289984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.592059:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828894.592060:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828894.592061:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:3.0:1787828894.592062:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828894.592063:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828894.592064:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828894.592066:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828894.592067:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:3.0:1787828894.592068:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828894.592069:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828894.592070:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:3.0:1787828894.592071:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828894.592072:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.592073:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828894.592075:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327290112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828894.592079:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828894.592080:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828894.592081:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828894.592083:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828894.592083:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828894.592084:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828894.592085:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828894.592086:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:3.0:1787828894.592088:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828894.592088:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828894.592089:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:3.0:1787828894.592091:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828894.592092:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.592093:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828894.592099:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828894.592101:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828894.592103:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828894.592104:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828895.292722:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297715373, at 4297715392
00000100:00000010:3.0:1787828895.292740:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f900.
00000100:00000040:2.0:1787828895.292755:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297715373, at 4297715392
00000100:00000010:2.0:1787828895.292766:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f4e0.
00000100:00000001:2.0:1787828899.196743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828899.196749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828899.196751:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828899.196755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828899.196759:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828899.196762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828899.196764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828899.196778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828899.196780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828899.708758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828899.708768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828899.708771:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828899.708775:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828899.708779:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828899.708786:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3047
00000400:00000200:3.0:1787828899.708795:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828899.708801:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828899.708805:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828899.708809:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.708812:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.708815:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.708817:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828899.708821:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828899.708823:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828899.708825:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828899.708829:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828899.708832:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828899.708834:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828899.708837:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.708839:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.708841:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.708843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828899.708846:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828899.708850:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828899.708853:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828899.708856:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828899.708859:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828899.708862:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.708864:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828899.708867:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828899.708876:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787828899.708880:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828899.708884:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828899.708885:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.708889:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828899.708891:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828899.708895:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828899.708903:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828899.708910:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828899.708920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327290240/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828899.708931:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828899.708937:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828899.708943:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327290240/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828899.708967:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828899.708971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828899.708974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828899.708975:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828899.708977:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828899.708980:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828899.708982:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3047
00000400:00000200:3.0:1787828899.708986:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828899.708989:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828899.708991:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828899.708996:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709000:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.709005:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709007:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828899.709010:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828899.709013:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828899.709015:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828899.709018:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828899.709022:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828899.709026:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828899.709031:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709034:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.709038:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709039:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828899.709043:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828899.709049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709051:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828899.709053:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828899.709057:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828899.709061:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709065:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828899.709070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828899.709075:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709077:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828899.709082:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787828899.709085:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828899.709088:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828899.709090:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709108:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828899.709113:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828899.709116:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828899.709121:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828899.709126:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828899.709134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327290368/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828899.709143:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828899.709147:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828899.709153:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327290368/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828899.709189:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828899.709193:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828899.709197:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828899.709199:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828899.709201:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828899.709206:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828899.709211:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3047
00000400:00000200:3.0:1787828899.709217:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828899.709219:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828899.709221:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828899.709226:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709229:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.709233:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709238:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828899.709241:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828899.709243:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828899.709246:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828899.709248:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828899.709252:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828899.709257:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828899.709262:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709266:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.709271:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709274:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828899.709279:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828899.709284:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709288:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828899.709290:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828899.709292:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828899.709297:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709300:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828899.709303:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828899.709307:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709310:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828899.709315:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828899.709318:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828899.709323:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828899.709324:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709327:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828899.709328:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828899.709331:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828899.709336:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787828899.709340:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828899.709344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:3.0:1787828899.709347:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327290496/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828899.709352:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828899.709354:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828899.709356:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327290496/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787828899.709365:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1787828899.709366:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828899.709372:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828899.709378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828899.709382:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828899.709384:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828899.709388:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828899.709390:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3047
00000400:00000200:3.0:1787828899.709395:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828899.709399:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828899.709402:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828899.709408:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709412:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.709417:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709419:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828899.709420:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828899.709423:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828899.709425:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828899.709427:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828899.709431:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828899.709435:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828899.709442:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709447:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828899.709450:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709454:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828899.709458:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828899.709464:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828899.709469:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828899.709473:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828899.709476:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828899.709479:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709481:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828899.709486:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828899.709488:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709490:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828899.709496:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828899.709501:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828899.709506:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828899.709507:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828899.709512:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828899.709513:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828899.709515:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828899.709522:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828899.709526:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828899.709532:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327290624/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828899.709543:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828899.709545:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828899.709550:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327290624/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828899.709562:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828899.709567:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3057)
00000800:00000200:0.1:1787828899.709796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828899.709798:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828899.709983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828899.709985:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787828899.710173:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828899.710175:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787828900.220775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828900.220783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828900.220785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828900.220789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828900.220793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828900.220797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828900.220799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828900.220803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828900.220805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828900.412740:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297720492, at 4297720512
00000100:00000040:3.0:1787828900.412740:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297720492, at 4297720512
00000100:00000010:2.0:1787828900.412758:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000100:00000010:3.0:1787828900.412762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000100:00080000:3.0:1787828904.828812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828904.828825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828904.828828:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828904.828833:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828904.828838:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828904.828846:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3052
00000400:00000200:3.0:1787828904.828855:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828904.828916:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828904.828921:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828904.828926:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.828930:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.828934:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.828949:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828904.828960:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828904.828963:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828904.828967:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828904.828971:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828904.828974:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828904.828976:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828904.828980:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.828983:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.828984:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.828986:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828904.828989:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828904.828993:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828904.828995:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828904.828998:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828904.829002:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828904.829004:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829006:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828904.829009:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828904.829032:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828904.829036:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828904.829040:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828904.829048:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829053:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828904.829069:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828904.829076:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828904.829113:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787828904.829119:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828904.829129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327290752/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828904.829141:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828904.829146:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828904.829150:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327290752/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828904.829173:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828904.829177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828904.829181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828904.829183:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828904.829185:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828904.829188:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828904.829191:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3052
00000400:00000200:3.0:1787828904.829196:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828904.829199:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828904.829202:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828904.829207:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829210:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.829212:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829225:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828904.829227:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828904.829230:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828904.829233:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828904.829238:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828904.829242:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828904.829243:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828904.829248:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829253:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.829257:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829261:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828904.829267:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828904.829272:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829278:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828904.829281:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828904.829287:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828904.829292:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829294:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828904.829300:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828904.829304:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829308:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828904.829316:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828904.829320:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828904.829329:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828904.829332:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829334:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828904.829337:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828904.829339:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828904.829345:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828904.829350:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828904.829355:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327290880/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828904.829362:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828904.829366:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828904.829370:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327290880/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828904.829383:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828904.829388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828904.829399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828904.829402:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828904.829406:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828904.829411:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828904.829415:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3052
00000400:00000200:3.0:1787828904.829420:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828904.829426:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828904.829428:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828904.829433:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829436:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.829437:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829440:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828904.829444:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828904.829446:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828904.829449:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828904.829450:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828904.829451:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828904.829453:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828904.829460:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829463:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.829466:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829469:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828904.829474:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828904.829478:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829481:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828904.829484:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828904.829487:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828904.829490:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829493:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828904.829498:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828904.829502:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829508:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828904.829514:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828904.829519:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828904.829532:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828904.829533:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829535:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828904.829537:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828904.829541:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828904.829547:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787828904.829551:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828904.829559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327291008/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828904.829569:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828904.829570:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828904.829578:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327291008/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828904.829591:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828904.829591:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787828904.829594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787828904.829596:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00080000:3.0:1787828904.829598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828904.829601:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828904.829603:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828904.829623:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828904.829631:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3052
00000400:00000200:3.0:1787828904.829637:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828904.829647:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828904.829652:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828904.829661:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829664:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.829694:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829697:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828904.829700:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828904.829702:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828904.829704:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828904.829706:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828904.829709:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828904.829714:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828904.829718:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829724:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828904.829727:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829742:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828904.829748:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828904.829756:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828904.829763:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828904.829767:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828904.829771:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828904.829778:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829781:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828904.829784:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828904.829793:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829798:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828904.829806:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828904.829812:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828904.829818:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828904.829822:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828904.829824:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828904.829826:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828904.829829:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828904.829841:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828904.829849:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828904.829859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327291136/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828904.829873:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828904.829876:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828904.829883:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327291136/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828904.829903:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828904.829913:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3062)
00000800:00000200:0.1:1787828904.830343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828904.830347:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787828904.830841:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828904.830845:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787828904.831321:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828904.831324:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000040:2.0:1787828905.532709:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297725613, at 4297725632
00000100:00000010:2.0:1787828905.532727:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000100:00000040:3.0:1787828905.532738:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297725613, at 4297725632
00000100:00000010:3.0:1787828905.532749:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000100:00000001:3.0:1787828909.948979:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828909.948983:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828909.948986:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828909.948994:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949005:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828909.949007:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949012:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828909.949015:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828909.949018:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828909.949037:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327291264:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828909.949042:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828909.949047:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828909.949050:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949052:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828909.949058:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828909.949060:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828909.949067:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828909.949071:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828909.949078:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828909.949086:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327291264, portal 25
00000100:00000001:3.0:1787828909.949089:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828909.949091:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:3.0:1787828909.949098:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949107:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828909.949110:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828909.949113:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828909.949117:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327291264, offset 0
00000400:00000200:3.0:1787828909.949124:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828909.949132:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.949138:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828909.949143:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828909.949148:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828909.949152:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828909.949155:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828909.949160:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40047:40047:256:4294967295] 192.168.203.116@tcp LPNI seq info [40047:40047:8:4294967295]
00000400:00000200:3.0:1787828909.949166:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.949177:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828909.949188:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.949194:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828909.949201:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828909.949206:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828909.949213:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828909.949217:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828909.949222:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.949225:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828909.949234:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828909.949239:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828909.949245:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828909.949248:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828909.949253:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828909.949257:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949269:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.949273:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949282:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949285:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828909.949290:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828909.949294:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828909.949298:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297730732
00000100:00000010:3.0:1787828909.949304:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f900.
00000100:00000001:3.0:1787828909.949310:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828909.949317:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297730732
00000100:00000010:3.0:1787828909.949320:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3ff00.
00000100:00000001:3.0:1787828909.949324:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828909.949328:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828909.949331:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949334:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828909.949339:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909]  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828909.949369:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828909.949371:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828909.949373:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828909.949376:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828909.949379:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828909.949382:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828909.949386:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828909.949398:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949406:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828909.949407:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949410:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.949412:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828909.949417:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949426:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828909.949431:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949440:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828909.949444:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949445:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.949447:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949450:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828909.949452:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828909.949455:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828909.949458:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828909.949460:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828909.949464:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828909.949471:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327291264:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828909.949474:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949476:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.949478:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327291264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949483:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828909.949485:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828909.949487:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828909.949489:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828909.949491:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828909.949492:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828909.949493:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828909.949495:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:3.0:1787828909.949498:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828909.949500:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828909.949501:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:3.0:1787828909.949503:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828909.949504:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949506:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949512:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828909.949513:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949515:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828909.949516:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828909.949517:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828909.949520:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949525:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828909.949527:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949528:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828909.949530:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828909.949531:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828909.949533:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291392:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828909.949536:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828909.949538:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828909.949539:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949540:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828909.949543:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828909.949544:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828909.949548:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828909.949549:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828909.949552:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828909.949555:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327291392, portal 10
00000100:00000001:3.0:1787828909.949557:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828909.949558:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:3.0:1787828909.949563:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949568:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828909.949570:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828909.949571:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828909.949573:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327291392, offset 0
00000400:00000200:3.0:1787828909.949576:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828909.949580:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.949582:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828909.949584:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828909.949587:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828909.949589:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828909.949591:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828909.949593:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40048:40048:256:4294967295] 192.168.203.116@tcp LPNI seq info [40048:40048:8:4294967295]
00000400:00000200:3.0:1787828909.949596:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.949601:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828909.949605:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.949607:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828909.949610:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828909.949612:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828909.949616:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828909.949618:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828909.949621:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.949623:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828909.949625:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828909.949628:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828909.949630:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828909.949632:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828909.949633:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828909.949636:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949642:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.949644:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949649:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949650:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828909.949653:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828909.949674:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828909.949677:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828909.949679:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297730059
00000100:00000001:3.0:1787828909.949682:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297730732
00000100:00000001:3.0:1787828909.949683:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297730059
00000100:00000001:3.0:1787828909.949685:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297730732
00000100:00000001:3.0:1787828909.949687:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828909.949688:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949690:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828909.949693:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909]  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828909.949698:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828909.949699:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828909.949701:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828909.949703:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828909.949704:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828909.949706:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828909.949708:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828909.949714:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949719:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828909.949720:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949722:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.949723:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828909.949725:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949729:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828909.949732:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949737:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828909.949738:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949739:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.949740:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949742:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828909.949743:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828909.949745:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828909.949747:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828909.949749:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828909.949752:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828909.949758:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291392:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828909.949761:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828909.949764:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949769:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828909.949770:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949771:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828909.949772:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828909.949774:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828909.949776:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291520:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828909.949778:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828909.949779:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828909.949781:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949782:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828909.949784:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3000.
02000000:00000001:3.0:1787828909.949786:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828909.949789:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828909.949790:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828909.949793:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828909.949795:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327291520, portal 4
00000100:00000001:3.0:1787828909.949796:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828909.949798:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:3.0:1787828909.949801:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949806:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828909.949808:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828909.949809:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828909.949811:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327291520, offset 0
00000400:00000200:3.0:1787828909.949814:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828909.949816:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.949819:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828909.949821:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828909.949823:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828909.949825:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828909.949827:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828909.949829:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40049:40049:256:4294967295] 192.168.203.116@tcp LPNI seq info [40049:40049:8:4294967295]
00000400:00000200:3.0:1787828909.949832:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.949836:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828909.949839:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.949841:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828909.949844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828909.949846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828909.949850:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828909.949852:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828909.949854:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.949856:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828909.949858:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828909.949861:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828909.949863:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828909.949864:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828909.949866:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828909.949868:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949874:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.949876:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949881:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949882:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828909.949884:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828909.949886:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828909.949888:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828909.949889:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297730066
00000100:00000001:3.0:1787828909.949891:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297730732
00000100:00000001:3.0:1787828909.949893:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297730066
00000100:00000001:3.0:1787828909.949894:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297730732
00000100:00000001:3.0:1787828909.949896:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828909.949898:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.949899:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828909.949902:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909]  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828909.949907:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828909.949907:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828909.949909:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828909.949910:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828909.949911:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828909.949912:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828909.949914:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828909.949920:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949925:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828909.949926:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949927:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.949928:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828909.949931:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949935:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828909.949937:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949942:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828909.949944:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949945:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.949945:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.949947:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828909.949948:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828909.949950:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828909.949953:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828909.949954:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828909.949957:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828909.949962:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291520:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828909.949964:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828909.949966:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.949971:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828909.949972:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949973:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828909.949974:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828909.949975:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828909.949977:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291648:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828909.949979:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828909.949980:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828909.949982:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828909.949983:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828909.949985:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828909.949987:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828909.949989:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828909.949990:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828909.949992:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828909.949994:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327291648, portal 4
00000100:00000001:3.0:1787828909.949996:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828909.949997:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:3.0:1787828909.950001:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950006:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828909.950008:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828909.950009:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828909.950011:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327291648, offset 0
00000400:00000200:3.0:1787828909.950014:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828909.950016:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.950018:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828909.950034:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828909.950037:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828909.950039:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828909.950040:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828909.950043:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40050:40050:256:4294967295] 192.168.203.116@tcp LPNI seq info [40050:40050:8:4294967295]
00000400:00000200:3.0:1787828909.950046:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828909.950051:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828909.950054:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.950056:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828909.950059:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828909.950061:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787828909.950064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828909.950066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787828909.950083:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828909.950085:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828909.950088:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828909.950090:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828909.950093:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828909.950094:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828909.950096:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828909.950099:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950105:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.950108:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950118:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.950119:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828909.950121:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828909.950124:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828909.950126:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828909.950128:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297730071
00000100:00000001:3.0:1787828909.950129:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297730732
00000100:00000001:3.0:1787828909.950131:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297730071
00000100:00000001:3.0:1787828909.950133:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297730732
00000100:00000001:3.0:1787828909.950134:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828909.950136:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.950137:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828909.950140:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909]  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828909.950145:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828909.950146:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828909.950147:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828909.950149:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828909.950150:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828909.950151:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828909.950154:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828909.950159:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950164:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828909.950165:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950166:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.950167:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828909.950169:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950175:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828909.950177:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950182:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828909.950184:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950185:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828909.950186:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950187:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828909.950188:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828909.950190:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828909.950192:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828909.950193:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828909.950196:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828909.950201:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291648:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828909.950203:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950205:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.950207:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327291392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950212:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828909.950213:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828909.950215:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828909.950216:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828909.950218:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828909.950219:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828909.950220:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828909.950222:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:3.0:1787828909.950224:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828909.950226:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828909.950227:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:3.0:1787828909.950229:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828909.950230:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950231:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.950233:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327291520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950238:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828909.950239:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828909.950240:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3000.
02000000:00000001:3.0:1787828909.950242:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828909.950242:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828909.950243:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828909.950244:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828909.950245:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:3.0:1787828909.950247:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828909.950248:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828909.950248:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:3.0:1787828909.950250:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828909.950251:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950252:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828909.950254:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327291648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828909.950258:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828909.950259:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828909.950260:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828909.950261:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828909.950262:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828909.950263:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828909.950264:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828909.950265:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:3.0:1787828909.950266:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828909.950267:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828909.950268:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:3.0:1787828909.950269:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828909.950271:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950272:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828909.950275:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828909.950276:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828909.950277:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828909.950279:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828910.652717:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297730732, at 4297730752
00000100:00000040:3.0:1787828910.652717:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297730732, at 4297730752
00000100:00000010:3.0:1787828910.652731:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3ff00.
00000100:00000010:2.0:1787828910.652734:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f900.
00000100:00000001:3.0:1787828915.068981:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828915.068997:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828915.069003:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828915.069024:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.069075:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828915.069080:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.069093:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828915.069097:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828915.069109:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828915.069132:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327291776:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828915.069149:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828915.069156:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828915.069160:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.069174:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828915.069186:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828915.069192:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828915.069208:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828915.069216:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828915.069226:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828915.069241:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327291776, portal 25
00000100:00000001:3.0:1787828915.069253:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828915.069258:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:3.0:1787828915.069268:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.069280:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828915.069287:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828915.069293:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828915.069301:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327291776, offset 0
00000400:00000200:3.0:1787828915.069315:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828915.069331:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.069346:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828915.069355:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828915.069368:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828915.069381:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828915.069396:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828915.069411:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40051:40051:256:4294967295] 192.168.203.116@tcp LPNI seq info [40051:40051:8:4294967295]
00000400:00000200:3.0:1787828915.069436:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.069469:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828915.069496:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.069507:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828915.069532:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828915.069543:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828915.069558:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828915.069565:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828915.069574:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.069578:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828915.069597:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828915.069616:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828915.069630:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828915.069640:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828915.069647:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828915.069683:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.069694:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.069700:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.069714:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.069720:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828915.069728:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828915.069735:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828915.069739:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297735853
00000100:00000010:3.0:1787828915.069745:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3ff00.
00000100:00000001:3.0:1787828915.069757:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828915.069768:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297735853
00000100:00000010:3.0:1787828915.069776:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f660.
00000100:00000001:3.0:1787828915.069785:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828915.069790:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828915.069799:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.069812:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828915.069827:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828915/real 1787828915]  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828915.069847:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828915.069853:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828915.069857:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828915.069868:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828915.069870:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828915.069874:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828915.069883:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828915.069925:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.069949:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828915.069953:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.069957:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.069959:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828915.069967:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.069977:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828915.069984:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070007:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828915.070011:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070013:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070015:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070018:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828915.070020:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828915.070024:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828915.070030:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828915.070033:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828915.070038:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828915.070048:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327291776:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828915.070054:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070058:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070061:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327291776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070068:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828915.070071:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070076:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828915.070079:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828915.070082:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828915.070084:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828915.070087:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828915.070090:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:3.0:1787828915.070095:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828915.070097:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828915.070099:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:3.0:1787828915.070103:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828915.070104:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070106:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070113:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828915.070115:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070117:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828915.070119:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828915.070120:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828915.070124:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070129:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828915.070130:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.070132:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828915.070134:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828915.070135:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828915.070137:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291904:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828915.070140:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828915.070141:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828915.070143:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.070144:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070146:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828915.070147:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828915.070150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828915.070153:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828915.070156:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828915.070158:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327291904, portal 10
00000100:00000001:3.0:1787828915.070160:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828915.070161:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:3.0:1787828915.070166:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070171:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828915.070173:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828915.070174:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828915.070176:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327291904, offset 0
00000400:00000200:3.0:1787828915.070179:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828915.070182:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.070185:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828915.070188:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828915.070189:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828915.070192:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828915.070193:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828915.070196:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40052:40052:256:4294967295] 192.168.203.116@tcp LPNI seq info [40052:40052:8:4294967295]
00000400:00000200:3.0:1787828915.070199:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.070204:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828915.070207:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.070209:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828915.070212:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828915.070215:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828915.070219:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828915.070221:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828915.070224:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.070225:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828915.070227:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828915.070230:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828915.070232:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828915.070234:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828915.070235:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828915.070238:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070244:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070246:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070251:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070252:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828915.070254:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828915.070257:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828915.070259:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828915.070261:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297735179
00000100:00000001:3.0:1787828915.070263:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297735853
00000100:00000001:3.0:1787828915.070265:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297735179
00000100:00000001:3.0:1787828915.070267:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297735853
00000100:00000001:3.0:1787828915.070268:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828915.070270:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070271:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828915.070273:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828915/real 1787828915]  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828915.070278:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828915.070279:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828915.070280:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828915.070281:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828915.070282:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828915.070284:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828915.070286:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828915.070291:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070295:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828915.070296:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070297:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070298:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828915.070300:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070304:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828915.070307:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070311:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828915.070313:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070315:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070316:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070317:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828915.070319:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828915.070321:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828915.070323:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828915.070324:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828915.070327:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828915.070331:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327291904:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828915.070334:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828915.070336:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070340:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828915.070341:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.070342:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828915.070344:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828915.070345:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828915.070346:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327292032:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828915.070348:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828915.070350:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828915.070350:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.070352:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070354:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a3000.
02000000:00000001:3.0:1787828915.070355:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828915.070358:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828915.070360:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828915.070362:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828915.070364:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327292032, portal 4
00000100:00000001:3.0:1787828915.070365:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828915.070367:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:3.0:1787828915.070371:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070375:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828915.070377:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828915.070378:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828915.070380:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327292032, offset 0
00000400:00000200:3.0:1787828915.070383:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828915.070385:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.070387:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828915.070389:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828915.070391:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828915.070392:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828915.070394:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828915.070396:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40053:40053:256:4294967295] 192.168.203.116@tcp LPNI seq info [40053:40053:8:4294967295]
00000400:00000200:3.0:1787828915.070399:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.070403:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828915.070406:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.070407:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828915.070410:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828915.070412:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828915.070415:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828915.070416:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828915.070419:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.070420:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828915.070422:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828915.070424:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828915.070426:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828915.070427:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828915.070429:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828915.070431:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070436:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070439:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070443:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070445:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828915.070446:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828915.070448:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828915.070450:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828915.070451:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297735186
00000100:00000001:3.0:1787828915.070454:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297735853
00000100:00000001:3.0:1787828915.070456:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297735186
00000100:00000001:3.0:1787828915.070459:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297735853
00000100:00000001:3.0:1787828915.070462:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828915.070464:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070466:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828915.070470:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828915/real 1787828915]  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828915.070478:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828915.070479:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828915.070481:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828915.070484:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828915.070486:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828915.070488:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828915.070492:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828915.070501:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070509:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828915.070510:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070512:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070514:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828915.070519:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070527:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828915.070531:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070535:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828915.070537:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070539:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070540:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070541:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828915.070542:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828915.070543:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828915.070545:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828915.070547:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828915.070549:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828915.070554:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327292032:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828915.070557:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828915.070560:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070564:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828915.070566:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.070567:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828915.070569:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828915.070570:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828915.070572:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327292160:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828915.070574:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828915.070576:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828915.070577:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828915.070578:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070581:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828915.070582:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828915.070585:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828915.070586:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828915.070589:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828915.070593:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327292160, portal 4
00000100:00000001:3.0:1787828915.070595:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828915.070596:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:3.0:1787828915.070603:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070610:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828915.070612:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828915.070615:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828915.070618:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327292160, offset 0
00000400:00000200:3.0:1787828915.070623:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828915.070627:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.070630:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828915.070633:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828915.070636:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828915.070640:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828915.070642:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828915.070644:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40054:40054:256:4294967295] 192.168.203.116@tcp LPNI seq info [40054:40054:8:4294967295]
00000400:00000200:3.0:1787828915.070647:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828915.070652:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828915.070675:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.070678:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828915.070681:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828915.070683:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787828915.070686:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828915.070688:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787828915.070690:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828915.070692:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828915.070695:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828915.070697:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828915.070699:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828915.070701:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828915.070702:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828915.070705:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070711:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070713:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070717:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070718:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828915.070720:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828915.070723:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828915.070725:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828915.070726:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297735192
00000100:00000001:3.0:1787828915.070728:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297735853
00000100:00000001:3.0:1787828915.070730:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297735192
00000100:00000001:3.0:1787828915.070732:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297735853
00000100:00000001:3.0:1787828915.070733:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828915.070734:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070735:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828915.070738:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828915/real 1787828915]  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828915.070743:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828915.070744:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828915.070745:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828915.070747:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828915.070748:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828915.070750:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828915.070752:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828915.070757:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070761:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828915.070762:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070763:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070764:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828915.070766:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070770:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828915.070772:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070776:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828915.070778:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070779:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828915.070780:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070781:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828915.070782:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828915.070783:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828915.070786:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828915.070787:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828915.070790:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828915.070795:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327292160:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828915.070797:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070798:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070801:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327291904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070805:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828915.070807:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070808:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828915.070809:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828915.070810:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828915.070811:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828915.070813:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828915.070814:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:3.0:1787828915.070816:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828915.070817:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828915.070818:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:3.0:1787828915.070820:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828915.070821:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070823:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070825:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327292032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070829:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828915.070829:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070830:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a3000.
02000000:00000001:3.0:1787828915.070832:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828915.070833:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828915.070833:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828915.070834:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828915.070835:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:3.0:1787828915.070836:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828915.070837:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828915.070838:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:3.0:1787828915.070839:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828915.070840:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070841:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828915.070842:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327292160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828915.070846:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828915.070848:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828915.070849:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a1800.
02000000:00000001:3.0:1787828915.070850:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828915.070851:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828915.070852:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828915.070853:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828915.070854:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:3.0:1787828915.070855:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828915.070856:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828915.070857:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:3.0:1787828915.070858:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828915.070859:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070860:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828915.070862:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828915.070863:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828915.070864:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828915.070865:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828915.580777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828915.580784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828915.580786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828915.580792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828915.580797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828915.580801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828915.580805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828915.580809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828915.580812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828915.772789:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297735853, at 4297735872
00000100:00000040:3.0:1787828915.772796:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297735853, at 4297735872
00000100:00000010:2.0:1787828915.772804:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3ff00.
00000100:00000010:3.0:1787828915.772810:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f660.
00000100:00000001:2.0:1787828916.604736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828916.604745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828916.604747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828916.604752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828916.604755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828916.604759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828916.604761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828916.604764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828916.604767:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828917.628746:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828917.628753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828917.628756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828917.628761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828917.628764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828917.628767:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828917.628770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828917.628772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828917.628774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828918.652750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828918.652758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828918.652760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828918.652765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828918.652769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828918.652772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828918.652775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828918.652777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828918.652780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828919.676751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828919.676759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828919.676761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828919.676766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828919.676770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828919.676773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828919.676776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828919.676786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828919.676789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828920.188881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828920.188889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828920.188892:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828920.188895:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828920.188900:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828920.188906:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3067
00000400:00000200:3.0:1787828920.188914:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828920.188921:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828920.188925:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828920.188928:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.188932:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.188935:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.188938:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828920.188942:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828920.188944:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828920.188948:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828920.188965:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828920.188969:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828920.188971:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828920.188974:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.188976:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.188977:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.188979:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828920.188981:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828920.188984:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828920.188986:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828920.188988:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828920.189002:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828920.189004:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189005:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828920.189007:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828920.189015:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787828920.189017:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828920.189020:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828920.189021:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189024:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828920.189026:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828920.189029:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828920.189035:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828920.189040:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828920.189047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327292288/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828920.189055:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828920.189058:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828920.189062:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327292288/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828920.189081:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828920.189084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828920.189086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828920.189087:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828920.189089:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828920.189091:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828920.189092:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3067
00000400:00000200:3.0:1787828920.189096:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828920.189098:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828920.189100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828920.189102:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189104:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.189106:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189108:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828920.189110:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828920.189112:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828920.189113:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828920.189115:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828920.189117:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828920.189118:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828920.189120:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189121:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.189122:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189124:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828920.189128:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828920.189142:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189143:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828920.189146:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828920.189152:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828920.189158:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189167:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828920.189179:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828920.189189:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189199:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828920.189212:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787828920.189226:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828920.189248:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828920.189256:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189261:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828920.189267:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828920.189273:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828920.189281:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828920.189293:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828920.189310:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327292416/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828920.189339:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828920.189342:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828920.189353:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327292416/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828920.189377:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828920.189389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828920.189400:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828920.189410:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828920.189419:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828920.189438:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828920.189448:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3067
00000400:00000200:3.0:1787828920.189472:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828920.189483:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828920.189496:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828920.189505:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189523:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.189546:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189555:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828920.189559:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828920.189563:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828920.189569:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828920.189578:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828920.189581:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828920.189583:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828920.189586:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189587:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.189589:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189598:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828920.189610:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828920.189628:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828920.189632:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828920.189641:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828920.189646:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828920.189649:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189651:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828920.189655:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828920.189661:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189666:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828920.189723:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828920.189727:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828920.189730:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828920.189730:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.189732:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828920.189736:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828920.189739:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828920.189753:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828920.189759:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828920.189773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327292544/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828920.189792:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828920.189797:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828920.189803:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327292544/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828920.189839:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828920.189855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787828920.189865:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787828920.189866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828920.189877:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1787828920.189877:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00080000:3.0:1787828920.189899:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828920.189913:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828920.189930:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3067
00000400:00000200:3.0:1787828920.189962:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828920.189976:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828920.189982:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828920.190010:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.190017:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.190025:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.190035:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828920.190046:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828920.190056:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828920.190062:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828920.190072:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828920.190076:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828920.190089:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828920.190093:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.190108:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828920.190120:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828920.190133:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828920.190146:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828920.190156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828920.190173:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828920.190177:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828920.190188:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828920.190201:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.190204:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828920.190207:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828920.190220:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.190226:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828920.190237:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828920.190245:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828920.190254:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828920.190258:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828920.190263:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828920.190267:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828920.190273:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828920.190281:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828920.190295:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828920.190317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327292672/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828920.190350:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828920.190361:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828920.190378:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327292672/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828920.190430:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828920.190441:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3077)
00000800:00000200:0.1:1787828920.190916:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828920.190919:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787828920.191347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828920.191349:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787828920.191674:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828920.191678:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000001:2.0:1787828920.700794:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828920.700800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828920.700802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828920.700805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828920.700808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828920.700811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828920.700813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828920.700815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828920.700817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828920.892780:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297740973, at 4297740992
00000100:00000040:3.0:1787828920.892785:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297740973, at 4297740992
00000100:00000010:2.0:1787828920.892796:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000100:00000010:3.0:1787828920.892799:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000001:2.0:1787828921.724782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828921.724793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828921.724795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828921.724802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828921.724807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828921.724811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828921.724826:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828921.724830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828921.724834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828922.748753:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828922.748759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828922.748761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828922.748764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828922.748767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828922.748770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828922.748773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828922.748776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828922.748778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828923.068844:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828923.068849:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828923.068854:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828923.772798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828923.772805:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828923.772808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828923.772815:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828923.772819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828923.772824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828923.772828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828923.772831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828923.772835:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828924.796813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828924.796821:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828924.796824:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828924.796829:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828924.796834:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828924.796839:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828924.796842:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828924.796844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828924.796848:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.309042:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828925.309050:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828925.309054:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828925.309074:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.309123:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828925.309130:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.309147:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828925.309152:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828925.309171:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828925.309186:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327292800:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828925.309192:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828925.309196:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828925.309200:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.309204:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828925.309215:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a2800.
02000000:00000001:3.0:1787828925.309219:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828925.309236:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828925.309242:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828925.309255:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828925.309272:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327292800, portal 25
00000100:00000001:3.0:1787828925.309276:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828925.309279:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:3.0:1787828925.309320:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.309415:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828925.309419:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828925.309427:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828925.309433:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327292800, offset 0
00000400:00000200:3.0:1787828925.309465:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828925.309492:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.309514:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828925.309532:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828925.309551:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828925.309563:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828925.309572:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828925.309602:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40059:40059:256:4294967295] 192.168.203.116@tcp LPNI seq info [40059:40059:8:4294967295]
00000400:00000200:3.0:1787828925.309619:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.309650:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828925.309669:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.309746:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828925.309762:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828925.309771:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828925.309790:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828925.309795:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828925.309820:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.309828:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828925.309847:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828925.309866:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828925.309879:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828925.309886:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828925.309897:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828925.309930:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.309980:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.309988:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310013:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310017:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828925.310026:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828925.310033:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828925.310039:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297746093
00000100:00000010:3.0:1787828925.310049:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3f660.
00000100:00000001:3.0:1787828925.310061:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787828925.310073:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297746093
00000100:00000010:3.0:1787828925.310080:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87e3fba0.
00000100:00000001:3.0:1787828925.310094:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787828925.310104:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828925.310111:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310115:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828925.310126:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828925/real 1787828925]  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828925.310144:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828925.310147:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828925.310150:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828925.310160:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828925.310163:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828925.310167:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828925.310174:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828925.310191:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310214:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828925.310219:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310224:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.310226:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828925.310239:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310265:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828925.310279:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310292:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828925.310296:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310298:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.310299:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310303:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828925.310305:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828925.310308:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828925.310312:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828925.310314:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828925.310318:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828925.310324:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327292800:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787828925.310327:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310329:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.310331:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327292800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310336:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828925.310338:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828925.310340:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a2800.
02000000:00000001:3.0:1787828925.310343:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828925.310345:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828925.310346:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:3.0:1787828925.310348:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828925.310350:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:3.0:1787828925.310353:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828925.310355:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828925.310356:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:3.0:1787828925.310359:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828925.310360:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310361:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310425:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828925.310427:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310429:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828925.310431:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787828925.310432:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828925.310435:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310446:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828925.310447:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.310449:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828925.310450:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828925.310452:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828925.310454:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327292928:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828925.310456:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828925.310458:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828925.310460:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.310461:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828925.310464:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a2800.
02000000:00000001:3.0:1787828925.310465:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828925.310468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828925.310470:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828925.310473:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828925.310475:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327292928, portal 10
00000100:00000001:3.0:1787828925.310477:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828925.310478:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:3.0:1787828925.310483:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310489:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828925.310492:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828925.310494:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828925.310497:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327292928, offset 0
00000400:00000200:3.0:1787828925.310502:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828925.310507:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.310512:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828925.310516:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828925.310519:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828925.310521:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828925.310524:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828925.310529:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40060:40060:256:4294967295] 192.168.203.116@tcp LPNI seq info [40060:40060:8:4294967295]
00000400:00000200:3.0:1787828925.310534:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.310542:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828925.310547:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.310550:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828925.310553:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828925.310555:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828925.310559:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828925.310561:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828925.310564:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.310565:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828925.310568:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828925.310571:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828925.310573:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828925.310575:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828925.310577:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828925.310579:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310585:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.310588:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310593:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310594:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828925.310596:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828925.310600:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:3.0:1787828925.310602:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828925.310604:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297745419
00000100:00000001:3.0:1787828925.310606:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297746093
00000100:00000001:3.0:1787828925.310608:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297745419
00000100:00000001:3.0:1787828925.310610:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297746093
00000100:00000001:3.0:1787828925.310611:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828925.310613:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310614:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828925.310617:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828925/real 1787828925]  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828925.310623:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828925.310624:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828925.310625:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828925.310626:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828925.310628:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828925.310629:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828925.310631:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828925.310638:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310642:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828925.310643:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310644:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.310645:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828925.310647:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310652:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828925.310678:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310687:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828925.310690:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310692:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.310695:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310697:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828925.310699:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828925.310703:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828925.310708:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828925.310710:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828925.310714:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828925.310720:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327292928:192.168.203.116@tcp:38:kworker.0
00000100:00000001:3.0:1787828925.310723:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828925.310726:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310731:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828925.310733:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.310735:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828925.310736:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828925.310738:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828925.310740:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327293056:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828925.310742:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828925.310744:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828925.310745:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.310747:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828925.310750:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0400.
02000000:00000001:3.0:1787828925.310751:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828925.310754:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828925.310757:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828925.310759:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828925.310762:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327293056, portal 4
00000100:00000001:3.0:1787828925.310763:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828925.310764:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:3.0:1787828925.310768:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310773:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828925.310775:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828925.310777:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828925.310778:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327293056, offset 0
00000400:00000200:3.0:1787828925.310781:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828925.310784:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.310787:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828925.310789:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828925.310791:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828925.310793:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828925.310795:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828925.310797:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40061:40061:256:4294967295] 192.168.203.116@tcp LPNI seq info [40061:40061:8:4294967295]
00000400:00000200:3.0:1787828925.310800:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.310805:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828925.310809:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.310811:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828925.310814:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828925.310816:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828925.310819:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828925.310821:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828925.310824:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.310826:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828925.310828:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828925.310831:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828925.310833:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828925.310834:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828925.310836:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828925.310838:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310844:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.310846:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310851:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310852:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828925.310855:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828925.310857:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:3.0:1787828925.310860:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828925.310861:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297745427
00000100:00000001:3.0:1787828925.310863:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297746093
00000100:00000001:3.0:1787828925.310864:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297745427
00000100:00000001:3.0:1787828925.310866:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297746093
00000100:00000001:3.0:1787828925.310868:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828925.310869:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.310871:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828925.310873:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828925/real 1787828925]  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828925.310878:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828925.310879:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828925.310880:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828925.310881:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828925.310882:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828925.310884:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828925.310886:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828925.310891:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310896:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828925.310897:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310899:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.310900:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828925.310904:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310911:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828925.310929:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310938:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828925.310941:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310943:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.310944:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.310946:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828925.310948:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828925.310951:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828925.310955:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828925.310959:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828925.310963:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828925.310972:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327293056:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828925.310975:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787828925.310978:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.310983:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787828925.310984:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.310986:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787828925.310987:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828925.310988:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828925.310990:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327293184:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828925.310993:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787828925.310994:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787828925.310996:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828925.310997:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787828925.311000:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828925.311001:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787828925.311004:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:3.0:1787828925.311006:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd91f98d10.
00000400:00000010:3.0:1787828925.311008:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99000.
00000100:00000200:3.0:1787828925.311010:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327293184, portal 4
00000100:00000001:3.0:1787828925.311012:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787828925.311013:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:3.0:1787828925.311017:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311022:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787828925.311023:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:3.0:1787828925.311025:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd91f99220.
00000100:00000200:3.0:1787828925.311026:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327293184, offset 0
00000400:00000200:3.0:1787828925.311029:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787828925.311032:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.311034:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:3.0:1787828925.311036:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787828925.311038:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787828925.311040:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787828925.311042:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787828925.311045:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40062:40062:256:4294967295] 192.168.203.116@tcp LPNI seq info [40062:40062:8:4294967295]
00000400:00000200:3.0:1787828925.311047:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828925.311052:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787828925.311056:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.311060:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd9288f000.
00000800:00000200:3.0:1787828925.311064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.0:1787828925.311067:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787828925.311072:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:3.1:1787828925.311075:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787828925.311079:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:3.0:1787828925.311081:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd9288f000.
00000400:00000200:3.0:1787828925.311085:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787828925.311089:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787828925.311092:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99220
00000400:00000010:3.0:1787828925.311094:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99220.
00000100:00000001:3.0:1787828925.311098:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787828925.311102:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311111:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.311114:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787828950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311119:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.311120:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787828925.311123:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828925.311125:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:3.0:1787828925.311127:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787828925.311129:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297745432
00000100:00000001:3.0:1787828925.311131:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297746093
00000100:00000001:3.0:1787828925.311132:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297745432
00000100:00000001:3.0:1787828925.311134:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297746093
00000100:00000001:3.0:1787828925.311135:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787828925.311136:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.311138:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787828925.311140:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828925/real 1787828925]  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787828925.311146:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787828925.311147:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787828925.311148:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd91f98d10.
00000400:00000200:3.0:1787828925.311150:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd91f99000
00000400:00000010:3.0:1787828925.311151:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd91f99000.
00000100:00000001:3.0:1787828925.311153:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787828925.311155:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787828925.311161:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311169:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787828925.311171:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311172:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.311173:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787828925.311175:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311180:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787828925.311182:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311186:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787828925.311188:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311189:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787828925.311190:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311191:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787828925.311193:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787828925.311195:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787828925.311197:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787828925.311199:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787828925.311201:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787828925.311206:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327293184:192.168.203.116@tcp:8:kworker.0
00000100:00000001:3.0:1787828925.311208:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311210:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.311212:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327292928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311216:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828925.311217:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828925.311219:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a2800.
02000000:00000001:3.0:1787828925.311221:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828925.311222:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828925.311223:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:3.0:1787828925.311225:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828925.311226:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:3.0:1787828925.311228:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828925.311229:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828925.311230:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:3.0:1787828925.311232:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828925.311233:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311234:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.311237:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327293056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311241:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828925.311242:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828925.311243:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0400.
02000000:00000001:3.0:1787828925.311244:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828925.311245:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828925.311246:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:3.0:1787828925.311247:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828925.311248:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:3.0:1787828925.311249:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828925.311250:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828925.311251:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:3.0:1787828925.311253:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828925.311253:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311254:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787828925.311256:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327293184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828925.311260:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787828925.311261:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787828925.311262:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd861a0800.
02000000:00000001:3.0:1787828925.311263:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787828925.311264:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787828925.311265:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:3.0:1787828925.311267:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787828925.311268:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:3.0:1787828925.311269:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787828925.311270:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787828925.311271:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:3.0:1787828925.311272:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787828925.311273:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311274:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787828925.311277:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787828925.311278:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828925.311279:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787828925.311280:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828925.820779:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828925.820785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828925.820787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828925.820790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828925.820794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828925.820796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828925.820798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828925.820800:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828925.820802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828926.012771:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297746093, at 4297746112
00000100:00000040:2.0:1787828926.012775:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297746093, at 4297746112
00000100:00000010:3.0:1787828926.012787:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3fba0.
00000100:00000010:2.0:1787828926.012789:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87e3f660.
00000020:00000001:3.0:1787828926.140850:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828926.140856:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828926.140860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828926.844755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828926.844761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828926.844763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828926.844767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828926.844770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828926.844773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828926.844775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828926.844788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828926.844791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828927.868776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828927.868783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828927.868786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828927.868792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828927.868797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828927.868801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828927.868805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828927.868808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828927.868812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828928.892761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828928.892768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828928.892771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828928.892791:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828928.892795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828928.892798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828928.892800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828928.892803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828928.892806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828929.212788:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828929.212793:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828929.212797:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828929.916728:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828929.916734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828929.916735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828929.916740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828929.916743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828929.916746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828929.916748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828929.916750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828929.916753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828930.428839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828930.428867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828930.428870:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828930.428876:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828930.428895:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828930.428904:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3078
00000400:00000200:3.0:1787828930.428916:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828930.428925:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828930.428946:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828930.428951:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.428955:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.428958:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.428961:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828930.428967:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828930.428970:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828930.428973:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828930.428977:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828930.428980:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828930.428983:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828930.428987:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.428989:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.428992:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.428994:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828930.428998:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828930.429003:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429006:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828930.429009:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828930.429013:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828930.429017:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429020:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828930.429023:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828930.429036:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828930.429040:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828930.429045:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828930.429047:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429051:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828930.429055:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828930.429061:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828930.429071:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828930.429079:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828930.429090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327293312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828930.429102:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828930.429108:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828930.429113:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327293312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828930.429150:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828930.429154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828930.429158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828930.429160:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828930.429162:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828930.429166:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828930.429170:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3078
00000400:00000200:3.0:1787828930.429175:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828930.429182:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828930.429188:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828930.429196:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429203:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.429208:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429215:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828930.429221:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828930.429227:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828930.429233:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828930.429245:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828930.429251:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828930.429258:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828930.429268:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429273:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.429278:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429285:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828930.429290:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828930.429297:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429301:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828930.429305:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828930.429314:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828930.429321:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429325:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828930.429332:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828930.429339:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429345:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828930.429354:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828930.429359:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828930.429366:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828930.429369:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429371:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828930.429375:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828930.429380:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828930.429388:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828930.429396:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828930.429408:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327293440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828930.429433:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828930.429438:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828930.429446:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327293440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828930.429460:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828930.429466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828930.429474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828930.429479:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828930.429482:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1787828930.429489:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828930.429490:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828930.429496:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3078
00000400:00000200:3.0:1787828930.429501:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828930.429505:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828930.429507:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828930.429514:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.1:1787828930.429516:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:3.0:1787828930.429520:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.429527:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429534:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828930.429539:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828930.429543:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828930.429548:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828930.429553:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828930.429557:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828930.429560:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828930.429568:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429573:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.429576:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429582:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828930.429587:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828930.429595:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429603:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828930.429606:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828930.429610:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828930.429616:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429622:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828930.429626:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828930.429633:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429636:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828930.429643:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828930.429648:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828930.429652:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828930.429697:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.429701:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828930.429713:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828930.429717:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828930.429724:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787828930.429731:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828930.429740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327293568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828930.429758:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828930.429765:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828930.429775:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327293568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828930.429793:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828930.429799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828930.429808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828930.429812:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828930.429816:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828930.429822:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828930.429829:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3078
00000400:00000200:3.0:1787828930.429839:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828930.429847:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828930.429853:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828930.429862:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.429868:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.429873:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.430062:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828930.430069:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828930.430073:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828930.430078:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828930.430083:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828930.430089:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828930.430095:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:0.1:1787828930.430101:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787828930.430105:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.1:1787828930.430105:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:3.0:1787828930.430112:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828930.430129:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828930.430135:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828930.430142:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828930.430151:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828930.430158:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828930.430164:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828930.430170:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828930.430178:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.430183:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828930.430188:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828930.430206:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.430212:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828930.430218:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828930.430225:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828930.430231:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828930.430234:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828930.430239:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828930.430243:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828930.430247:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828930.430255:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828930.430262:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828930.430274:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327293696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828930.430305:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828930.430311:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828930.430324:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327293696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828930.430350:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828930.430359:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3088)
00000800:00000200:0.1:1787828930.430477:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828930.430479:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828930.430769:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828930.430771:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787828930.940766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828930.940789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828930.940791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828930.940795:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828930.940799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828930.940802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828930.940804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828930.940807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828930.940810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828931.132754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297751213, at 4297751232
00000100:00000040:3.0:1787828931.132761:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297751213, at 4297751232
00000100:00000010:2.0:1787828931.132768:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000100:00000010:3.0:1787828931.132775:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000100:00000001:2.0:1787828931.964729:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828931.964734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828931.964735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828931.964739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828931.964742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828931.964745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828931.964747:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828931.964749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828931.964751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828932.284811:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828932.284818:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828932.284824:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828932.988761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828932.988766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828932.988768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828932.988772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828932.988774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828932.988777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828932.988779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828932.988781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828932.988783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828934.012730:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828934.012736:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828934.012737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828934.012741:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828934.012744:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828934.012747:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828934.012749:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828934.012750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828934.012753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828935.036785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828935.036791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828935.036792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828935.036796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828935.036799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828935.036802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828935.036804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828935.036806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828935.036810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828935.356821:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828935.356826:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828935.356829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828935.548727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828935.548736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828935.548738:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828935.548742:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828935.548751:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828935.548757:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3083
00000400:00000200:3.0:1787828935.548766:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828935.548772:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828935.548776:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828935.548780:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548782:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.548785:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548787:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828935.548790:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828935.548791:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828935.548794:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828935.548797:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828935.548799:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828935.548801:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828935.548805:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548808:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.548811:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548814:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828935.548819:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828935.548823:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548827:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828935.548832:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828935.548836:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828935.548839:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.548856:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828935.548860:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828935.548868:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787828935.548873:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828935.548876:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828935.548878:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.548880:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828935.548882:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828935.548887:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828935.548894:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828935.548901:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828935.548913:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327293824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828935.548926:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828935.548930:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828935.548937:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327293824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828935.548960:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828935.548963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828935.548966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828935.548968:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828935.548969:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828935.548971:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828935.548973:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3083
00000400:00000200:3.0:1787828935.548978:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828935.548981:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828935.548983:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828935.548986:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548988:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.548990:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.548993:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828935.548997:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828935.548999:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828935.549003:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828935.549007:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828935.549009:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828935.549013:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828935.549022:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549026:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.549030:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549035:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828935.549039:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828935.549044:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549048:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828935.549055:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828935.549062:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828935.549069:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549073:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828935.549078:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828935.549084:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549088:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828935.549092:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787828935.549098:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828935.549101:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828935.549104:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549106:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828935.549108:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828935.549110:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828935.549116:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828935.549121:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828935.549133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327293952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828935.549149:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828935.549154:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828935.549162:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327293952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828935.549177:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828935.549191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828935.549198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828935.549201:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828935.549203:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828935.549209:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828935.549213:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3083
00000400:00000200:3.0:1787828935.549219:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828935.549226:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828935.549232:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828935.549235:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549236:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.549238:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549240:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828935.549241:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828935.549242:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828935.549244:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828935.549246:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828935.549253:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828935.549258:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828935.549264:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549270:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.549276:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549282:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828935.549289:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828935.549300:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549306:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828935.549310:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828935.549315:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828935.549322:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549326:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828935.549328:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828935.549331:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549336:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828935.549342:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828935.549345:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828935.549349:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828935.549353:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549358:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828935.549359:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828935.549362:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828935.549366:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787828935.549371:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828935.549378:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327294080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828935.549390:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828935.549395:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828935.549400:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327294080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828935.549413:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787828935.549414:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787828935.549417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1787828935.549421:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00080000:3.0:1787828935.549422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828935.549425:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828935.549429:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828935.549433:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828935.549438:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3083
00000400:00000200:3.0:1787828935.549445:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828935.549449:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828935.549452:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828935.549458:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549461:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.549466:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549471:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828935.549474:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828935.549475:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828935.549477:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828935.549479:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828935.549483:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828935.549485:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828935.549492:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549495:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828935.549499:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549504:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828935.549507:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828935.549514:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828935.549519:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828935.549522:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828935.549525:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828935.549529:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549533:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828935.549536:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828935.549542:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549546:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828935.549553:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828935.549558:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828935.549563:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828935.549566:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828935.549572:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828935.549577:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828935.549581:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828935.549588:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828935.549595:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828935.549606:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327294208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828935.549627:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828935.549632:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828935.549642:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327294208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828935.549682:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828935.549690:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3093)
00000800:00000200:1.1:1787828935.549918:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828935.549920:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787828935.550127:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828935.550129:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787828935.550330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828935.550332:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1787828936.060750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828936.060756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828936.060759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828936.060765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828936.060769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828936.060773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828936.060776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828936.060778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828936.060782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828936.252740:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297756332, at 4297756352
00000100:00000040:3.0:1787828936.252750:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297756332, at 4297756352
00000100:00000010:2.0:1787828936.252754:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e639c0.
00000100:00000010:3.0:1787828936.252760:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63000.
00000100:00000001:2.0:1787828937.084785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828937.084793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828937.084796:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828937.084802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828937.084806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828937.084810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828937.084814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828937.084830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828937.084849:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828938.108716:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828938.108723:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828938.108725:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828938.108731:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828938.108735:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828938.108738:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828938.108740:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828938.108742:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828938.108744:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828938.428840:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828938.428845:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828938.428849:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828939.132717:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828939.132722:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828939.132723:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828939.132728:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828939.132731:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828939.132733:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828939.132735:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828939.132737:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828939.132739:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828940.156725:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828940.156730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828940.156732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828940.156737:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828940.156740:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828940.156743:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828940.156746:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828940.156748:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828940.156750:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828940.669372:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828940.669377:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828940.669846:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828940.669848:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828940.670040:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828940.670041:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787828940.670255:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828940.670257:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787828941.180753:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828941.180759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828941.180762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828941.180767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828941.180771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828941.180775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828941.180778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828941.180781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828941.180784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828941.372722:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297761452, at 4297761472
00000100:00000010:2.0:1787828941.372736:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000040:3.0:1787828941.372751:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297761452, at 4297761472
00000100:00000010:3.0:1787828941.372762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000020:00000001:3.0:1787828941.500820:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828941.500824:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828941.500827:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828942.204726:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828942.204732:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828942.204734:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828942.204738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828942.204740:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828942.204744:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828942.204746:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828942.204748:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828942.204750:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828943.228704:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828943.228709:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828943.228711:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828943.228714:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828943.228716:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828943.228719:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828943.228720:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828943.228722:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828943.228725:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828944.252800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828944.252806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828944.252807:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828944.252811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828944.252814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828944.252817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828944.252819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828944.252821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828944.252823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828944.572776:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828944.572792:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828944.572795:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828945.276740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828945.276748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828945.276750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828945.276756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828945.276760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828945.276765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828945.276768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828945.276771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828945.276787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828945.788870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828945.788881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828945.788885:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828945.788889:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828945.788894:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828945.788902:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3093
00000400:00000200:3.0:1787828945.788912:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828945.788918:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828945.788922:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828945.788925:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.788944:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.788947:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.788949:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828945.788953:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828945.788955:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828945.788958:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828945.788961:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828945.788963:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828945.788966:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828945.788968:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.788970:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.788971:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.788973:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828945.788976:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828945.788979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828945.788981:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828945.788984:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828945.788987:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828945.788990:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.788992:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828945.788994:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828945.789004:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828945.789008:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828945.789012:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828945.789013:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789017:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828945.789020:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828945.789041:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828945.789050:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787828945.789058:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828945.789070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327294848/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828945.789082:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828945.789087:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828945.789092:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327294848/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828945.789122:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828945.789126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828945.789130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828945.789132:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828945.789135:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828945.789138:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828945.789142:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3093
00000400:00000200:3.0:1787828945.789146:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828945.789151:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828945.789154:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828945.789158:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789160:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.789162:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789165:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828945.789171:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828945.789174:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828945.789180:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828945.789186:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828945.789192:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828945.789201:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828945.789216:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789223:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.789230:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789238:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828945.789246:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828945.789255:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789278:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828945.789283:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828945.789290:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828945.789297:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789302:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828945.789308:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828945.789316:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789322:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828945.789330:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828945.789336:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828945.789343:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828945.789345:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789350:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828945.789355:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828945.789360:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828945.789369:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828945.789377:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828945.789389:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327294976/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828945.789402:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828945.789405:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828945.789412:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327294976/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828945.789431:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828945.789435:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828945.789444:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828945.789448:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828945.789452:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828945.789461:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828945.789475:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3093
00000400:00000200:3.0:1787828945.789486:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828945.789496:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828945.789502:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828945.789508:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789528:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.789539:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789548:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828945.789557:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828945.789562:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828945.789566:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828945.789573:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828945.789579:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828945.789587:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828945.789598:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789620:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787828945.789622:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787828945.789626:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.1:1787828945.789629:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:3.0:1787828945.789631:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828945.789637:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828945.789644:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828945.789652:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828945.789655:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828945.789660:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828945.789665:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789690:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828945.789696:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828945.789784:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789791:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828945.789801:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828945.789825:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828945.789831:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828945.789833:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.789838:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828945.789843:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828945.789849:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828945.789858:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787828945.789864:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828945.789875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327295104/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828945.789891:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828945.789896:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828945.789906:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327295104/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828945.789927:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828945.789933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828945.789959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828945.789964:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828945.789970:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828945.789978:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828945.789987:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3093
00000400:00000200:3.0:1787828945.789999:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828945.790011:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828945.790017:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828945.790028:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.790036:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.790044:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.790053:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828945.790061:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828945.790067:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828945.790073:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828945.790079:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828945.790088:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828945.790096:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828945.790107:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.790113:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828945.790122:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828945.790131:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828945.790137:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828945.790144:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828945.790152:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828945.790157:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828945.790163:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828945.790170:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.790176:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828945.790182:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828945.790189:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.790192:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828945.790198:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828945.790206:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828945.790212:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828945.790216:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828945.790221:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828945.790225:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828945.790229:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828945.790237:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828945.790260:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828945.790271:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327295232/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828945.790288:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828945.790292:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828945.790302:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327295232/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828945.790338:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828945.790350:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3103)
00000800:00000200:1.1:1787828945.790442:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828945.790445:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787828945.790829:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828945.790831:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787828945.791194:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828945.791196:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787828946.300748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828946.300753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828946.300755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828946.300759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828946.300761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828946.300764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828946.300783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828946.300786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828946.300788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828946.492800:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297766573, at 4297766592
00000100:00000010:2.0:1787828946.492815:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d20.
00000100:00000040:3.0:1787828946.492829:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297766573, at 4297766592
00000100:00000010:3.0:1787828946.492849:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63480.
00000100:00000001:2.0:1787828947.324722:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828947.324728:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828947.324729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828947.324733:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828947.324736:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828947.324738:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828947.324740:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828947.324742:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828947.324744:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828947.644977:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828947.644984:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828947.644991:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828948.348733:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828948.348738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828948.348740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828948.348744:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828948.348747:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828948.348750:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828948.348752:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828948.348766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828948.348769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828949.372772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828949.372778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828949.372780:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828949.372785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828949.372789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828949.372793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828949.372795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828949.372798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828949.372800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828950.396767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828950.396772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828950.396774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828950.396777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828950.396797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828950.396804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828950.396807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828950.396810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828950.396813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828950.716900:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828950.716904:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828950.716908:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787828950.908763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828950.908773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828950.908777:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828950.908781:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828950.908785:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828950.908792:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3098
00000400:00000200:3.0:1787828950.908801:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828950.908808:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828950.908811:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828950.908814:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908822:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.908824:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908826:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828950.908829:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828950.908831:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828950.908834:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828950.908837:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828950.908839:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828950.908840:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828950.908843:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908845:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.908846:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908848:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828950.908850:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828950.908853:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908855:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828950.908857:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828950.908860:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828950.908863:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.908865:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828950.908866:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828950.908878:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787828950.908881:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828950.908885:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828950.908886:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.908888:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828950.908892:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828950.908897:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828950.908905:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828950.908912:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828950.908921:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327295360/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828950.908930:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828950.908934:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828950.908943:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327295360/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828950.908963:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828950.908966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828950.908969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828950.908970:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828950.908971:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828950.908974:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828950.908975:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3098
00000400:00000200:3.0:1787828950.908978:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828950.908980:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828950.908983:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828950.908985:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908987:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.908989:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.908991:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828950.908994:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828950.908995:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828950.908996:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828950.908998:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828950.909003:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828950.909009:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828950.909018:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909024:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.909036:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909041:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828950.909047:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828950.909057:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909063:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828950.909065:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828950.909072:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828950.909077:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909079:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828950.909084:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828950.909089:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909095:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828950.909104:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787828950.909110:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828950.909115:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828950.909120:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909125:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828950.909130:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828950.909134:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828950.909148:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828950.909154:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828950.909163:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327295488/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828950.909170:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828950.909175:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828950.909180:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327295488/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828950.909198:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828950.909203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828950.909210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828950.909217:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828950.909222:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828950.909230:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828950.909236:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3098
00000400:00000200:3.0:1787828950.909253:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828950.909262:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828950.909269:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828950.909279:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787828950.909280:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828950.909287:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787828950.909287:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000040:3.0:1787828950.909293:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909298:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828950.909302:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828950.909305:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828950.909308:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828950.909312:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828950.909313:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828950.909315:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828950.909318:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909323:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.909326:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909331:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828950.909336:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828950.909341:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909344:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828950.909348:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828950.909354:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828950.909360:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909366:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828950.909371:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828950.909376:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909382:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828950.909392:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828950.909398:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828950.909401:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828950.909403:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909407:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828950.909409:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828950.909413:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828950.909419:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787828950.909423:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828950.909439:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327295616/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828950.909451:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828950.909455:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828950.909463:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327295616/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828950.909473:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828950.909478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828950.909485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828950.909488:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828950.909490:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828950.909493:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828950.909497:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3098
00000400:00000200:3.0:1787828950.909504:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828950.909508:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828950.909520:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828950.909524:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909530:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.909532:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909536:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828950.909540:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828950.909543:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828950.909546:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828950.909549:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828950.909554:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828950.909558:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828950.909566:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909571:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828950.909574:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909577:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828950.909578:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828950.909580:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828950.909587:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828950.909591:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828950.909595:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828950.909615:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909618:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828950.909622:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828950.909627:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909632:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828950.909637:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828950.909642:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828950.909648:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828950.909650:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828950.909655:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828950.909658:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828950.909661:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828950.909666:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828950.909694:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828950.909700:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327295744/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828950.909712:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828950.909716:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828950.909721:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327295744/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828950.909760:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828950.909767:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3108)
00000800:00000200:1.1:1787828950.909945:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828950.909947:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787828950.910184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828950.910202:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787828950.910466:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828950.910468:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1787828951.420749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828951.420758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828951.420760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828951.420764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828951.420767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828951.420770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828951.420773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828951.420775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828951.420778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828951.612757:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297771692, at 4297771712
00000100:00000010:3.0:1787828951.612773:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63b40.
00000100:00000040:2.0:1787828951.612780:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297771692, at 4297771712
00000100:00000010:2.0:1787828951.612793:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d80.
00000100:00000001:2.0:1787828952.444797:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828952.444803:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828952.444804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828952.444808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828952.444811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828952.444814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828952.444816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828952.444818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828952.444820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828953.468747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828953.468755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828953.468757:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828953.468763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828953.468766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828953.468769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828953.468772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828953.468775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828953.468778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828953.788849:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828953.788855:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828953.788860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828954.492768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828954.492773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828954.492774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828954.492779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828954.492781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828954.492784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828954.492786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828954.492789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828954.492791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828955.516730:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828955.516738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828955.516740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828955.516745:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828955.516748:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828955.516750:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828955.516752:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828955.516754:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828955.516757:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828956.028769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828956.028782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828956.028785:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828956.028791:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828956.028796:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828956.028804:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3103
00000400:00000200:3.0:1787828956.028813:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828956.028830:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828956.028834:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828956.028837:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.028840:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.028842:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.028844:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828956.028848:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828956.028850:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828956.028853:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828956.028857:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828956.028858:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828956.028860:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828956.028863:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.028864:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.028866:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.028867:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828956.028870:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828956.028873:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828956.028875:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828956.028877:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828956.028880:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828956.028883:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.028884:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828956.028886:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828956.028898:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828956.028901:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828956.028905:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828956.028907:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.028910:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828956.028914:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828956.028919:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828956.028928:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787828956.028935:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828956.028945:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327295872/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828956.028966:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828956.028970:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828956.028973:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327295872/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828956.028993:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828956.028996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828956.028999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828956.029004:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828956.029005:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828956.029008:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828956.029010:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3103
00000400:00000200:3.0:1787828956.029013:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828956.029015:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828956.029016:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828956.029019:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029020:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.029022:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029023:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828956.029024:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828956.029025:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828956.029026:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828956.029028:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828956.029030:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828956.029033:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828956.029036:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029037:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.029039:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029041:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828956.029042:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828956.029044:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029045:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828956.029046:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828956.029059:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828956.029063:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029064:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828956.029069:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828956.029072:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029076:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828956.029079:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828956.029082:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828956.029085:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828956.029088:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029091:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828956.029094:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828956.029096:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828956.029102:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828956.029107:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828956.029114:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327296000/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828956.029136:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828956.029139:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828956.029151:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327296000/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828956.029164:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828956.029168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828956.029171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828956.029175:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828956.029178:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828956.029182:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828956.029186:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3103
00000400:00000200:3.0:1787828956.029193:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828956.029197:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828956.029200:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828956.029203:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029204:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.029210:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029213:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828956.029215:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828956.029217:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828956.029221:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828956.029229:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828956.029232:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828956.029234:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828956.029239:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029241:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.029245:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029248:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828956.029252:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828956.029258:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029262:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828956.029267:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828956.029270:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828956.029274:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029277:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828956.029281:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828956.029286:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029287:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828956.029291:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828956.029294:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828956.029296:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828956.029297:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029302:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828956.029305:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828956.029307:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828956.029311:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787828956.029313:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828956.029320:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327296128/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828956.029335:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828956.029339:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828956.029344:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327296128/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828956.029355:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828956.029360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828956.029365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828956.029367:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828956.029371:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828956.029375:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828956.029379:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3103
00000400:00000200:3.0:1787828956.029394:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828956.029399:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828956.029402:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828956.029405:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029408:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.029410:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029415:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828956.029421:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828956.029424:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000200:1.1:1787828956.029425:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:3.0:1787828956.029426:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828956.029430:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828956.029433:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828956.029435:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828956.029437:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029439:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828956.029440:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029441:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828956.029443:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828956.029444:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828956.029445:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828956.029447:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828956.029449:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828956.029451:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029456:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828956.029460:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828956.029468:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029472:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828956.029488:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828956.029495:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828956.029501:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828956.029505:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828956.029510:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828956.029515:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828956.029520:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828956.029531:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828956.029549:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828956.029561:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327296256/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828956.029601:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828956.029618:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828956.029632:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327296256/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828956.029686:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828956.029693:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3113)
00000800:00000200:1.1:1787828956.029815:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828956.029961:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787828956.030112:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787828956.540776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828956.540784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828956.540787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828956.540793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828956.540797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828956.540801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828956.540804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828956.540808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828956.540811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828956.732717:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297776812, at 4297776832
00000100:00000010:3.0:1787828956.732733:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a20.
00000100:00000040:2.0:1787828956.732859:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297776812, at 4297776832
00000100:00000010:2.0:1787828956.732873:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f00.
00000020:00000001:3.0:1787828956.860834:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828956.860837:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828956.860840:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828957.564813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828957.564821:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828957.564823:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828957.564828:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828957.564832:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828957.564836:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828957.564839:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828957.564842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828957.564846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828958.588741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828958.588748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828958.588750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828958.588754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828958.588757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828958.588760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828958.588762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828958.588765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828958.588769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828959.612764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828959.612769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828959.612771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828959.612775:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828959.612778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828959.612781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828959.612783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828959.612785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828959.612788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787828959.932831:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787828959.932835:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787828959.932838:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787828960.636787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828960.636793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828960.636795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828960.636799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828960.636802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828960.636822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828960.636824:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828960.636826:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828960.636829:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828961.148758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828961.148768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828961.148771:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828961.148775:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828961.148780:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828961.148788:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3108
00000400:00000200:3.0:1787828961.148797:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828961.148805:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828961.148808:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828961.148812:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148814:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.148816:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148818:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828961.148822:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828961.148823:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828961.148826:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828961.148829:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828961.148831:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828961.148833:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828961.148836:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148837:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.148838:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148839:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828961.148842:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828961.148844:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148846:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828961.148848:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828961.148851:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828961.148854:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.148856:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828961.148857:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828961.148865:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787828961.148869:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828961.148874:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828961.148876:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.148880:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828961.148883:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828961.148889:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828961.148898:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828961.148907:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828961.148919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327296384/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828961.148931:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828961.148936:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828961.148942:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327296384/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828961.148964:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828961.148967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828961.148970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828961.148971:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828961.148972:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828961.148975:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828961.148976:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3108
00000400:00000200:3.0:1787828961.148979:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828961.148981:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828961.148984:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828961.148986:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148989:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.148992:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.148995:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828961.148998:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828961.148999:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828961.149002:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828961.149004:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828961.149007:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828961.149009:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828961.149013:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149015:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.149018:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149020:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828961.149024:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828961.149029:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149032:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828961.149037:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828961.149047:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828961.149051:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149058:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828961.149066:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828961.149073:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149077:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828961.149082:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787828961.149087:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828961.149092:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828961.149097:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149101:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828961.149105:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828961.149109:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828961.149116:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787828961.149123:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828961.149133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327296512/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828961.149146:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828961.149150:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828961.149160:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327296512/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828961.149176:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828961.149182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828961.149187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828961.149191:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828961.149195:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828961.149200:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828961.149206:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3108
00000400:00000200:3.0:1787828961.149215:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828961.149223:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828961.149229:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828961.149238:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149243:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.149248:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149254:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828961.149259:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828961.149263:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828961.149268:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828961.149272:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828961.149275:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828961.149279:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828961.149288:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149293:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.149300:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149304:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828961.149311:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828961.149333:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149341:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828961.149346:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828961.149352:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828961.149357:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149360:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828961.149366:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828961.149373:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149376:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828961.149385:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787828961.149391:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828961.149398:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828961.149399:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149402:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828961.149406:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828961.149410:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828961.149418:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000800:00000200:0.1:1787828961.149424:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828961.149425:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828961.149430:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00100000:3.0:1787828961.149436:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327296640/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828961.149462:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828961.149469:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828961.149477:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327296640/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828961.149492:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828961.149498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828961.149505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828961.149509:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828961.149512:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828961.149519:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828961.149615:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3108
00000400:00000200:3.0:1787828961.149626:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828961.149634:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828961.149641:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828961.149648:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149709:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.149716:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149723:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828961.149729:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828961.149732:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828961.149736:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828961.149741:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828961.149745:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828961.149750:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828961.149759:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149764:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828961.149771:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149776:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828961.149782:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828961.149789:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828961.149794:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828961.149800:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828961.149805:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828961.149811:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149817:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828961.149821:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828961.149828:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149834:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828961.149843:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787828961.149850:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828961.149856:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828961.149860:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828961.149866:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828961.149870:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828961.149875:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828961.149881:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828961.149889:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828961.149900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327296768/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828961.149917:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828961.149922:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828961.149934:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327296768/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828961.149949:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828961.149957:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3118)
00000800:00000200:0.1:1787828961.150171:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828961.150174:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828961.150440:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828961.150442:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787828961.150624:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828961.150626:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787828961.660795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828961.660803:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828961.660805:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828961.660811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828961.660821:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828961.660825:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828961.660827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828961.660830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828961.660832:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828961.852738:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297781932, at 4297781952
00000100:00000010:2.0:1787828961.852761:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000100:00000040:3.0:1787828961.852764:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297781932, at 4297781952
00000100:00000010:3.0:1787828961.852773:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000100:00000001:2.0:1787828962.684739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828962.684745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828962.684746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828962.684750:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828962.684753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828962.684755:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828962.684758:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828962.684760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828962.684762:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828963.708756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828963.708761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828963.708762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828963.708766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828963.708769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828963.708772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828963.708774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828963.708776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828963.708778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828964.732770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828964.732777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828964.732779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828964.732784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828964.732798:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828964.732801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828964.732804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828964.732807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828964.732810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828965.756745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828965.756752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828965.756753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828965.756758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828965.756761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828965.756763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828965.756766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828965.756768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828965.756770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828966.268778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828966.268792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828966.268797:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828966.268803:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828966.268809:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828966.268818:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3113
00000400:00000200:3.0:1787828966.268830:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828966.268838:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828966.268842:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828966.268845:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.268849:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.268852:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.268855:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828966.268861:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828966.268863:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828966.268867:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828966.268872:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828966.268874:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828966.268877:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828966.268881:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.268883:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.268885:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.268887:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828966.268891:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828966.268894:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828966.268897:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828966.268901:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828966.268905:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828966.268909:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.268912:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828966.268915:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828966.268927:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787828966.268932:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828966.268938:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828966.268940:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.268944:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828966.268947:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828966.268953:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828966.268963:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787828966.268972:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828966.268985:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327296896/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828966.268998:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828966.269004:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828966.269010:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327296896/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828966.269038:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828966.269040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828966.269043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828966.269044:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828966.269046:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828966.269048:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828966.269050:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3113
00000400:00000200:3.0:1787828966.269053:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828966.269056:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828966.269058:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828966.269060:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269063:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.269065:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269071:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828966.269073:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828966.269075:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828966.269076:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828966.269078:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828966.269081:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828966.269083:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828966.269090:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269092:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.269096:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269100:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828966.269107:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828966.269109:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269111:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828966.269113:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828966.269121:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828966.269128:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269135:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828966.269141:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828966.269150:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269156:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828966.269166:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787828966.269172:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828966.269175:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828966.269180:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269182:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828966.269184:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828966.269187:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828966.269195:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787828966.269202:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828966.269211:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327297024/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828966.269227:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828966.269234:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828966.269239:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327297024/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828966.269258:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828966.269263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828966.269268:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828966.269272:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828966.269275:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828966.269278:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828966.269285:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3113
00000400:00000200:3.0:1787828966.269297:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828966.269301:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828966.269305:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828966.269311:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269313:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.269315:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269317:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828966.269326:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828966.269328:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828966.269330:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828966.269336:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828966.269339:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828966.269345:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828966.269359:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269362:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.269370:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269373:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828966.269380:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828966.269387:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269393:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828966.269400:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828966.269405:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828966.269421:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269427:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828966.269433:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828966.269439:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269448:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828966.269455:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828966.269463:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828966.269469:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828966.269472:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269479:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828966.269482:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828966.269484:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828966.269490:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787828966.269498:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828966.269506:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327297152/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828966.269520:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828966.269523:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828966.269532:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327297152/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828966.269547:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828966.269550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828966.269560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828966.269563:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828966.269566:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828966.269572:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828966.269579:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3113
00000400:00000200:3.0:1787828966.269593:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828966.269599:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828966.269600:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828966.269604:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269605:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.269607:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269609:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828966.269615:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828966.269618:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828966.269622:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828966.269625:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828966.269641:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828966.269647:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828966.269682:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269684:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828966.269687:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269688:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828966.269690:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828966.269697:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828966.269700:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828966.269705:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828966.269709:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828966.269711:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269714:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828966.269717:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828966.269721:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269723:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828966.269728:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787828966.269731:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828966.269738:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828966.269740:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828966.269744:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828966.269747:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828966.269751:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828966.269755:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000800:00000200:0.1:1787828966.269761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787828966.269763:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828966.269769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327297280/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828966.269774:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828966.269780:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828966.269785:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000100:00000040:3.0:1787828966.269786:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327297280/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828966.269800:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828966.269809:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3124)
00000800:00000200:0.1:1787828966.270296:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828966.270299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787828966.270685:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828966.270688:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787828966.271020:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828966.271024:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000100:00000001:2.0:1787828966.780786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828966.780794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828966.780796:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828966.780802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828966.780805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828966.780810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828966.780812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828966.780815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828966.780819:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828966.972742:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297787053, at 4297787072
00000100:00000040:2.0:1787828966.972746:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297787053, at 4297787072
00000100:00000010:3.0:1787828966.972759:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000100:00000010:2.0:1787828966.972765:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000001:2.0:1787828967.804778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828967.804804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828967.804806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828967.804811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828967.804815:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828967.804819:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828967.804822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828967.804825:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828967.804827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828968.828727:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828968.828735:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828968.828737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828968.828742:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828968.828745:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828968.828750:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828968.828752:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828968.828755:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828968.828758:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828969.852810:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828969.852819:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828969.852822:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828969.852828:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828969.852833:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828969.852837:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828969.852841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828969.852844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828969.852848:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828970.876798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828970.876811:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828970.876815:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828970.876825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828970.876833:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828970.876839:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828970.876843:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828970.876849:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828970.876855:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828971.388739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828971.388747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787828971.388750:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828971.388753:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828971.388757:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828971.388762:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3119
00000400:00000200:3.0:1787828971.388769:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828971.388774:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828971.388777:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828971.388779:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388781:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.388783:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388785:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828971.388787:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828971.388789:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787828971.388791:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828971.388794:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787828971.388795:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787828971.388797:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787828971.388800:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388801:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.388803:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388804:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828971.388806:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787828971.388808:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388810:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828971.388813:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828971.388817:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828971.388820:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.388822:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787828971.388824:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828971.388835:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787828971.388839:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787828971.388843:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828971.388845:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.388848:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828971.388851:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828971.388855:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828971.388863:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787828971.388869:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828971.388877:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327297408/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828971.388885:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828971.388888:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828971.388892:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327297408/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828971.388910:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828971.388912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828971.388915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787828971.388916:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828971.388917:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828971.388920:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828971.388921:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3119
00000400:00000200:3.0:1787828971.388924:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828971.388926:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828971.388928:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828971.388930:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388932:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.388933:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388935:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828971.388936:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828971.388937:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787828971.388939:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828971.388940:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787828971.388942:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787828971.388944:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787828971.388947:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388949:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.388951:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388953:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828971.388955:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787828971.388958:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828971.388961:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828971.388963:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828971.388969:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828971.388972:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.388974:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828971.388979:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828971.388983:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.388988:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828971.388995:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787828971.388999:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787828971.389004:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828971.389006:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389009:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828971.389012:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828971.389016:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828971.389021:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787828971.389026:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828971.389034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327297536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828971.389055:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828971.389058:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828971.389063:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327297536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828971.389079:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828971.389082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828971.389085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787828971.389089:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828971.389090:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828971.389092:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828971.389095:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3119
00000400:00000200:3.0:1787828971.389101:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828971.389108:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828971.389113:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828971.389122:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389128:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.389135:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389141:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828971.389146:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828971.389150:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787828971.389153:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828971.389155:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787828971.389161:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787828971.389164:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787828971.389172:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389179:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.389184:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389191:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828971.389198:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787828971.389208:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389212:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828971.389216:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828971.389220:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828971.389226:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389233:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828971.389237:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828971.389244:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389250:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828971.389257:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:3.0:1787828971.389258:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000800:00000200:0.1:1787828971.389260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000020:00000040:3.0:1787828971.389264:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787828971.389271:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828971.389273:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389276:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828971.389279:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828971.389281:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828971.389287:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787828971.389293:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828971.389301:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327297664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828971.389324:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828971.389330:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828971.389339:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327297664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828971.389357:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787828971.389359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787828971.389363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787828971.389364:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787828971.389365:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787828971.389370:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787828971.389375:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3119
00000400:00000200:3.0:1787828971.389380:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787828971.389385:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787828971.389387:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787828971.389391:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389393:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.389395:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389397:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787828971.389401:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787828971.389403:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787828971.389404:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787828971.389407:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787828971.389411:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787828971.389413:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787828971.389417:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389419:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787828971.389422:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389424:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787828971.389428:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787828971.389430:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787828971.389432:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787828971.389434:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787828971.389440:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787828971.389444:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389448:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787828971.389452:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787828971.389458:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389461:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787828971.389468:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787828971.389473:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787828971.389479:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787828971.389483:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787828971.389486:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787828971.389489:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787828971.389492:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787828971.389498:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787828971.389502:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787828971.389512:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327297792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787828971.389524:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787828971.389527:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828971.389534:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327297792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787828971.389561:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828971.389570:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3129)
00000800:00000200:0.1:1787828971.389785:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828971.389787:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787828971.390014:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828971.390017:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787828971.390221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828971.390223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00000001:2.0:1787828971.900741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828971.900746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828971.900747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828971.900751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828971.900754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828971.900757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828971.900759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828971.900761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828971.900763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787828972.092718:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297792172, at 4297792192
00000100:00000040:2.0:1787828972.092725:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297792172, at 4297792192
00000100:00000010:3.0:1787828972.092731:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000100:00000010:2.0:1787828972.092734:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000100:00000001:2.0:1787828972.924733:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828972.924738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828972.924739:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828972.924743:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828972.924747:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828972.924749:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828972.924753:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828972.924755:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828972.924757:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828973.948788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828973.948796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828973.948798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828973.948809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828973.948814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828973.948818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828973.948822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828973.948825:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828973.948828:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828974.972704:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828974.972711:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828974.972713:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828974.972718:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828974.972721:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828974.972725:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828974.972728:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828974.972730:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828974.972733:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828975.996776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828975.996785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828975.996788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828975.996794:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828975.996798:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828975.996803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828975.996806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828975.996809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828975.996812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828976.509347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828976.509352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787828976.510010:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828976.510012:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787828976.510286:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828976.510288:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787828976.510494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828976.510495:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:2.0:1787828977.020770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828977.020777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828977.020779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828977.020785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828977.020789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828977.020793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828977.020796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828977.020800:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828977.020804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828977.212716:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297797292, at 4297797312
00000100:00000040:3.0:1787828977.212716:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297797292, at 4297797312
00000100:00000010:2.0:1787828977.212729:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000100:00000010:3.0:1787828977.212732:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000100:00000001:2.0:1787828978.044757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828978.044763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828978.044765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828978.044769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828978.044772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828978.044775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828978.044777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828978.044779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828978.044781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828979.068746:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787828979.068753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787828979.068756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828979.068761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787828979.068764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787828979.068769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828979.068772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787828979.068774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787828979.068776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787828981.629471:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828981.629499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828981.630348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828981.630351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828981.630581:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828981.630583:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787828981.630846:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828981.630848:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1787828982.332732:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297802412, at 4297802432
00000100:00000040:3.0:1787828982.332738:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297802413, at 4297802432
00000100:00000010:2.0:1787828982.332746:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000010:3.0:1787828982.332748:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00080000:2.0:1787828986.748780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828986.748789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787828986.748791:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828986.748794:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828986.748798:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828986.748805:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3134
00000400:00000200:2.0:1787828986.748813:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828986.748818:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828986.748821:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828986.748823:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748825:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.748827:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748829:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828986.748832:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828986.748833:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787828986.748835:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828986.748838:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787828986.748839:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787828986.748841:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787828986.748843:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748844:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.748846:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748847:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828986.748849:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787828986.748851:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748854:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828986.748856:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828986.748859:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828986.748861:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.748863:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787828986.748864:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828986.748873:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787828986.748875:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787828986.748879:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828986.748880:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.748882:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828986.748884:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828986.748888:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828986.748894:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787828986.748899:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828986.748906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327298944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828986.748914:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828986.748917:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828986.748921:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327298944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828986.748936:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828986.748939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828986.748941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787828986.748942:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828986.748943:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828986.748945:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828986.748947:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3134
00000400:00000200:2.0:1787828986.748950:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828986.748953:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828986.748955:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828986.748957:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748960:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.748964:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.748966:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828986.748970:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828986.748973:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787828986.748975:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828986.748977:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787828986.748980:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787828986.748984:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787828986.748993:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749000:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.749006:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749013:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828986.749021:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787828986.749030:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749037:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828986.749041:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828986.749051:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828986.749056:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749061:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828986.749067:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828986.749075:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749081:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828986.749088:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787828986.749095:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787828986.749099:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828986.749101:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749104:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828986.749106:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828986.749110:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828986.749116:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787828986.749123:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828986.749132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327299072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828986.749146:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828986.749152:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828986.749173:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327299072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828986.749200:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828986.749207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828986.749216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787828986.749221:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1787828986.749222:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828986.749226:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00080000:2.0:1787828986.749231:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828986.749239:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828986.749247:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3134
00000400:00000200:2.0:1787828986.749259:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828986.749269:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828986.749286:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828986.749295:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749300:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.749305:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749316:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828986.749324:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828986.749326:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787828986.749333:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828986.749337:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787828986.749341:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787828986.749347:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787828986.749353:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749359:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.749364:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749368:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828986.749374:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787828986.749381:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749385:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828986.749387:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828986.749392:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828986.749396:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749399:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828986.749403:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828986.749408:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749413:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828986.749420:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787828986.749425:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787828986.749431:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828986.749433:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749437:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828986.749440:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828986.749442:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828986.749449:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787828986.749455:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828986.749466:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327299200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828986.749479:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828986.749483:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828986.749509:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327299200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828986.749525:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787828986.749533:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787828986.749538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787828986.749542:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787828986.749545:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787828986.749551:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787828986.749556:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3134
00000400:00000200:2.0:1787828986.749564:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787828986.749570:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787828986.749573:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787828986.749581:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749585:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.749589:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749592:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787828986.749595:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787828986.749599:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787828986.749601:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787828986.749604:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787828986.749609:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787828986.749615:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787828986.749619:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749622:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787828986.749627:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749630:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787828986.749635:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787828986.749640:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787828986.749642:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787828986.749646:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787828986.749650:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787828986.749686:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749691:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787828986.749695:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787828986.749700:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749704:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787828986.749712:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787828986.749718:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787828986.749725:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787828986.749729:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787828986.749736:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787828986.749740:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787828986.749744:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787828986.749749:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787828986.749757:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787828986.749767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327299328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787828986.749781:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787828986.749783:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828986.749789:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327299328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787828986.749800:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787828986.749806:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3144)
00000800:00000200:0.1:1787828986.749874:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828986.749876:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828986.750067:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828986.750069:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787828986.750297:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828986.750298:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:3.0:1787828987.452755:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297807532, at 4297807552
00000100:00000040:2.0:1787828987.452759:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297807532, at 4297807552
00000100:00000010:3.0:1787828987.452771:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:2.0:1787828987.452775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000800:00000200:0.1:1787828991.869269:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828991.869277:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828991.869713:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828991.869716:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828991.869947:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828991.869949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787828991.870124:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828991.870126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787828992.572734:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297812652, at 4297812672
00000100:00000040:3.0:1787828992.572739:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297812652, at 4297812672
00000100:00000010:2.0:1787828992.572749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000100:00000010:3.0:1787828992.572752:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000800:00000200:0.1:1787828996.989313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828996.989341:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828996.989879:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828996.989881:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828996.990076:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828996.990078:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787828996.990259:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787828996.990261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:3.0:1787828997.692786:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297817772, at 4297817792
00000100:00000040:2.0:1787828997.692793:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297817772, at 4297817792
00000100:00000010:3.0:1787828997.692811:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000010:2.0:1787828997.692816:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000020:00000001:3.0:1787829001.916768:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829001.916772:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829001.916775:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829002.108808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829002.108820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829002.108823:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829002.108827:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829002.108833:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829002.108841:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3149
00000400:00000200:2.0:1787829002.108852:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829002.108859:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829002.108864:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829002.108868:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.108872:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.108875:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.108888:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829002.108893:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829002.108896:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829002.108898:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829002.108902:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829002.108904:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829002.108907:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829002.108910:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.108913:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.108915:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.108917:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829002.108920:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829002.108923:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829002.108926:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829002.108929:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829002.108933:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829002.108936:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.108939:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829002.108941:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829002.108950:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829002.108954:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829002.108958:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829002.108960:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.108963:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829002.108966:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829002.108970:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829002.108977:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829002.108984:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829002.108994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327300480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829002.109005:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829002.109010:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829002.109015:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327300480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829002.109036:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829002.109039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829002.109042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829002.109044:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829002.109046:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829002.109048:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829002.109051:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3149
00000400:00000200:2.0:1787829002.109055:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829002.109058:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829002.109061:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829002.109065:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109067:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.109070:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109072:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829002.109075:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829002.109076:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829002.109078:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829002.109081:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829002.109086:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829002.109091:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829002.109107:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109114:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.109119:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109123:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829002.109128:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829002.109135:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109140:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829002.109146:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829002.109154:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829002.109161:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109165:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829002.109171:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829002.109177:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109181:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829002.109188:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787829002.109194:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829002.109200:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829002.109202:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109207:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829002.109211:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829002.109215:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829002.109224:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829002.109231:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829002.109241:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327300608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829002.109265:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829002.109268:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829002.109275:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327300608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829002.109288:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829002.109295:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829002.109301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829002.109304:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829002.109307:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829002.109312:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829002.109320:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3149
00000400:00000200:2.0:1787829002.109325:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829002.109331:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829002.109336:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829002.109342:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109347:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.109351:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109354:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829002.109358:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829002.109359:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829002.109363:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829002.109367:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829002.109372:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829002.109393:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829002.109402:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109409:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.109415:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109418:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829002.109422:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829002.109438:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109444:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829002.109448:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829002.109453:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829002.109459:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109465:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829002.109468:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829002.109474:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829002.109478:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829002.109479:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829002.109482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000010:2.0:1787829002.109486:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787829002.109493:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829002.109498:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829002.109503:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109508:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829002.109512:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829002.109516:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829002.109525:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829002.109533:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829002.109546:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327300736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829002.109560:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829002.109566:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829002.109573:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327300736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829002.109603:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829002.109610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829002.109621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829002.109626:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829002.109631:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829002.109639:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829002.109647:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3149
00000400:00000200:2.0:1787829002.109699:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829002.109710:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829002.109716:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829002.109725:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109729:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.109734:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109739:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829002.109745:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829002.109749:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829002.109754:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829002.109759:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829002.109765:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829002.109770:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829002.109779:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109785:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829002.109792:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109797:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829002.109804:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829002.109812:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829002.109817:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829002.109822:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829002.109828:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829002.109835:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109840:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829002.109845:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829002.109850:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109856:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829002.109865:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787829002.109872:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829002.109879:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829002.109884:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829002.109891:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829002.109896:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829002.109901:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829002.109909:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829002.109917:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829002.109929:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327300864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829002.109949:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829002.109956:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829002.109968:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327300864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829002.109989:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829002.109998:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3159)
00000800:00000200:0.1:1787829002.110052:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829002.110054:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829002.110258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829002.110260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829002.110555:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829002.110557:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787829002.812752:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297822892, at 4297822912
00000100:00000040:3.0:1787829002.812752:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297822892, at 4297822912
00000100:00000010:3.0:1787829002.812766:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000100:00000010:2.0:1787829002.812768:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000020:00000001:3.0:1787829004.988899:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829004.988902:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829004.988906:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787829007.229263:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829007.229267:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787829007.229945:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829007.229949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787829007.230151:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829007.230153:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787829007.230420:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829007.230422:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000100:00000040:2.0:1787829007.932736:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297828012, at 4297828032
00000100:00000040:3.0:1787829007.932756:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297828012, at 4297828032
00000100:00000010:2.0:1787829007.932758:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00000100:00000010:3.0:1787829007.932773:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000020:00000001:3.0:1787829008.060816:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829008.060821:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829008.060825:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829011.132851:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829011.132857:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829011.132862:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829012.156784:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829012.156788:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829012.156792:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787829012.350047:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829012.350070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787829012.350586:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829012.350588:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787829012.350885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829012.350887:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787829012.351132:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829012.351134:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000100:00000040:2.0:1787829013.052806:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297833133, at 4297833152
00000100:00000040:3.0:1787829013.052815:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297833133, at 4297833152
00000100:00000010:2.0:1787829013.052827:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000100:00000010:3.0:1787829013.052840:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000020:00000001:3.0:1787829014.204797:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829014.204800:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829014.204803:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829015.228868:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829015.228873:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829015.228878:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829017.276858:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829017.276864:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829017.276869:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787829017.469448:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829017.469455:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829017.470210:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829017.470212:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829017.470515:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829017.470517:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829017.470848:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829017.470851:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000040:2.0:1787829018.172727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297838252, at 4297838272
00000100:00000010:2.0:1787829018.172741:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000040:3.0:1787829018.172753:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297838253, at 4297838272
00000100:00000010:3.0:1787829018.172762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000020:00000001:3.0:1787829018.300841:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829018.300846:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829018.300850:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829020.348836:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829020.348841:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829020.348844:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829021.372831:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829021.372837:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829021.372841:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829022.588774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829022.588785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829022.588787:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829022.588791:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829022.588796:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829022.588802:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3170
00000400:00000200:2.0:1787829022.588821:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829022.588828:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829022.588831:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829022.588834:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.588836:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.588838:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.588840:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829022.588844:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829022.588846:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829022.588848:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829022.588852:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829022.588854:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829022.588856:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829022.588858:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.588860:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.588861:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.588862:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829022.588864:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829022.588867:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829022.588869:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829022.588871:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829022.588875:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829022.588877:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.588879:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829022.588880:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829022.588891:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829022.588893:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829022.588898:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829022.588899:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.588902:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829022.588905:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829022.588909:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829022.588921:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829022.588931:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829022.588943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327302528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829022.588956:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829022.588961:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829022.588967:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327302528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829022.588990:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829022.588993:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829022.588996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829022.588997:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829022.588998:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829022.589001:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829022.589003:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3170
00000400:00000200:2.0:1787829022.589007:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829022.589009:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829022.589011:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829022.589013:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589015:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.589018:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589020:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829022.589022:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829022.589023:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829022.589024:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829022.589026:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829022.589028:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829022.589029:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829022.589032:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589042:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.589052:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589058:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829022.589067:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829022.589080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589088:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829022.589092:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829022.589100:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829022.589113:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589125:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829022.589136:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829022.589146:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589150:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829022.589160:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829022.589169:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829022.589182:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829022.589186:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589191:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829022.589195:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829022.589200:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829022.589208:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829022.589218:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829022.589231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327302656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829022.589264:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829022.589269:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829022.589275:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327302656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829022.589286:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829022.589289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829022.589294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829022.589301:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829022.589304:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829022.589307:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829022.589316:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3170
00000400:00000200:2.0:1787829022.589327:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829022.589336:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829022.589340:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829022.589344:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589357:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.589364:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589372:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829022.589380:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829022.589383:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829022.589388:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829022.589395:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829022.589399:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829022.589404:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829022.589414:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589426:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.589433:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589440:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829022.589445:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829022.589453:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589458:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829022.589463:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829022.589469:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.1:1787829022.589475:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829022.589477:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589484:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829022.589487:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829022.589491:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589493:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829022.589497:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787829022.589502:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829022.589512:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829022.589520:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589529:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829022.589533:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829022.589538:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829022.589548:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829022.589559:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829022.589574:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327302784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829022.589619:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829022.589625:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829022.589638:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327302784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829022.589753:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829022.589759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829022.589768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829022.589771:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829022.589773:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829022.589781:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829022.589788:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3170
00000400:00000200:2.0:1787829022.589800:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829022.589810:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829022.589816:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829022.589827:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589838:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.589848:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589854:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829022.589859:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829022.589862:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829022.589866:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829022.589870:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829022.589882:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829022.589889:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829022.589898:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589905:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829022.589913:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589923:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829022.589931:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829022.589941:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829022.589950:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829022.589956:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829022.589963:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829022.589973:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589976:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829022.589978:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829022.589984:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.589991:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829022.589998:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787829022.590004:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829022.590011:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829022.590015:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829022.590022:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829022.590029:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829022.590034:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829022.590043:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829022.590053:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829022.590068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327302912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829022.590100:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829022.590108:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829022.590121:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327302912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829022.590152:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829022.590163:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829022.590182:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3180)
00000800:00000200:0.1:1787829022.590401:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829022.590552:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829023.292716:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297843372, at 4297843392
00000100:00000040:2.0:1787829023.292717:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297843372, at 4297843392
00000100:00000010:3.0:1787829023.292733:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000100:00000010:2.0:1787829023.292736:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000020:00000001:2.0:1787829023.420899:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829023.420907:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829023.420913:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829024.444848:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829024.444851:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829024.444854:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829026.492835:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829026.492840:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829026.492844:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829027.516793:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829027.516798:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829027.516801:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787829027.709606:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829027.709613:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829027.710322:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829027.710325:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829027.710603:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829027.710605:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829027.710938:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829027.710941:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1787829028.412735:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297848493, at 4297848512
00000100:00000040:3.0:1787829028.412736:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297848493, at 4297848512
00000100:00000010:2.0:1787829028.412750:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:3.0:1787829028.412753:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000020:00000001:2.0:1787829029.564825:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829029.564829:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829029.564831:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829030.588822:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829030.588827:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829030.588830:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829032.636896:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829032.636899:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829032.636903:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829032.828960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829032.828972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829032.828976:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829032.828980:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829032.828986:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829032.828994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3180
00000400:00000200:2.0:1787829032.829004:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829032.829011:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829032.829015:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829032.829019:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829023:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.829025:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829029:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829032.829033:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829032.829036:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829032.829039:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829032.829043:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829032.829045:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829032.829048:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829032.829052:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829054:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.829057:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829059:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829032.829063:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829032.829068:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829071:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829032.829074:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829032.829078:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829032.829081:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.829083:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829032.829086:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829032.829097:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787829032.829101:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829032.829106:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829032.829108:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.829111:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829032.829114:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829032.829119:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829032.829127:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829032.829135:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829032.829146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327303552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829032.829158:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829032.829163:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829032.829202:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327303552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829032.829234:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829032.829238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829032.829242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829032.829244:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829032.829246:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829032.829250:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829032.829255:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3180
00000400:00000200:2.0:1787829032.829261:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829032.829268:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829032.829271:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829032.829282:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829287:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.829294:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829301:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829032.829305:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829032.829312:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829032.829317:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829032.829322:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829032.829334:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829032.829346:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829032.829373:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829388:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.829398:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829407:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829032.829417:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829032.829429:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829447:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829032.829454:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829032.829464:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829032.829477:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.829484:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829032.829492:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829032.829498:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.829509:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829032.829518:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829032.829532:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829032.829553:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829032.829561:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.829569:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829032.829574:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829032.829580:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829032.829592:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829032.829602:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829032.829616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327303680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829032.829651:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829032.829716:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829032.829729:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327303680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829032.829769:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829032.829779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829032.829799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829032.829809:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829032.829813:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829032.829824:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829032.829835:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3180
00000400:00000200:2.0:1787829032.829852:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829032.829857:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829032.829861:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829032.829869:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829878:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.829892:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829902:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829032.829906:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829032.829908:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829032.829917:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829032.829927:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829032.829932:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829032.829943:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829032.829950:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829959:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.829970:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.829977:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829032.829984:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787829032.829991:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787829032.829996:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.1:1787829032.830004:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829032.830006:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829032.830021:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829032.830028:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829032.830033:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.830038:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829032.830042:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829032.830062:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.830072:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829032.830090:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787829032.830105:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829032.830123:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829032.830128:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.830138:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829032.830141:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829032.830147:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829032.830161:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829032.830189:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829032.830215:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327303808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829032.830230:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829032.830235:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829032.830249:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327303808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829032.830274:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829032.830280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829032.830289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829032.830291:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829032.830294:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829032.830307:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829032.830319:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3180
00000400:00000200:2.0:1787829032.830335:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829032.830350:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829032.830359:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829032.830381:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.830391:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.830401:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.830412:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829032.830426:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829032.830433:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829032.830442:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829032.830449:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829032.830465:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829032.830476:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829032.830499:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.830506:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829032.830516:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829032.830528:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829032.830545:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829032.830555:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829032.830566:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829032.830580:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829032.830591:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829032.830596:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.830600:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829032.830609:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829032.830626:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.830631:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829032.830645:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787829032.830652:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829032.830690:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829032.830693:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829032.830699:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829032.830703:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829032.830707:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829032.830719:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829032.830728:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829032.830754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327303936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829032.830854:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829032.830863:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829032.830881:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327303936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829032.830936:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829032.830946:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3190)
00000800:00000200:0.1:1787829032.831115:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829032.831117:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829032.831345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829032.831347:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829032.831582:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829032.831584:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787829033.532766:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297853613, at 4297853632
00000100:00000040:3.0:1787829033.532785:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297853613, at 4297853632
00000100:00000010:2.0:1787829033.532786:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000010:3.0:1787829033.532807:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000020:00000001:3.0:1787829033.660880:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829033.660884:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829033.660888:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829035.708833:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829035.708836:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829035.708839:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829036.732819:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829036.732824:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829036.732828:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829037.948739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829037.948749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829037.948752:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829037.948756:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829037.948761:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829037.948769:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3185
00000400:00000200:2.0:1787829037.948779:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829037.948787:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829037.948792:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829037.948796:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948800:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.948803:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948806:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829037.948812:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829037.948814:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829037.948817:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829037.948822:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829037.948825:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829037.948827:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829037.948831:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948833:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.948834:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948836:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829037.948838:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829037.948841:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948843:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829037.948845:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829037.948849:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829037.948851:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.948853:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829037.948855:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829037.948865:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829037.948868:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829037.948872:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829037.948874:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.948877:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829037.948880:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829037.948895:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829037.948904:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829037.948911:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829037.948929:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327304064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829037.948938:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829037.948942:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829037.948946:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327304064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829037.948970:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829037.948973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829037.948976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829037.948977:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829037.948978:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829037.948981:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829037.948983:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3185
00000400:00000200:2.0:1787829037.948986:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829037.948989:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829037.948991:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829037.948993:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948995:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.948996:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.948998:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829037.949000:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829037.949001:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829037.949002:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829037.949004:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829037.949006:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829037.949007:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829037.949010:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949012:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.949015:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949018:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829037.949020:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829037.949023:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949027:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829037.949028:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829037.949037:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829037.949042:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949045:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829037.949050:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829037.949054:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949057:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829037.949061:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787829037.949064:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829037.949068:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829037.949070:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949072:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829037.949074:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829037.949077:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829037.949082:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829037.949087:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829037.949106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327304192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829037.949115:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829037.949119:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829037.949124:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327304192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829037.949143:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829037.949147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829037.949151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829037.949153:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829037.949155:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829037.949158:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829037.949161:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3185
00000400:00000200:2.0:1787829037.949166:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829037.949180:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829037.949182:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829037.949187:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949188:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.949190:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949192:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829037.949194:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829037.949196:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829037.949199:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829037.949202:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829037.949206:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829037.949207:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829037.949212:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949214:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.949216:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949218:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829037.949221:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829037.949224:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949228:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829037.949230:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829037.949233:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829037.949236:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949240:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829037.949243:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829037.949246:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949250:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829037.949257:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787829037.949260:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829037.949262:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829037.949264:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949267:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829037.949268:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829037.949269:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829037.949275:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829037.949279:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829037.949286:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327304320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829037.949293:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829037.949296:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829037.949305:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327304320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829037.949315:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829037.949320:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829037.949323:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829037.949325:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829037.949327:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829037.949330:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829037.949334:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3185
00000400:00000200:2.0:1787829037.949339:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829037.949343:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829037.949346:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829037.949349:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949353:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.949355:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949359:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829037.949364:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829037.949367:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829037.949369:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829037.949371:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829037.949372:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829037.949374:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829037.949380:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949383:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829037.949387:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949391:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829037.949393:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829037.949397:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829037.949400:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829037.949403:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829037.949405:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829037.949407:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949408:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829037.949410:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829037.949414:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949415:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829037.949418:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787829037.949421:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829037.949424:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829037.949425:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829037.949428:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829037.949431:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829037.949432:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829037.949435:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829037.949440:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829037.949446:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327304448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.1:1787829037.949448:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829037.949456:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829037.949460:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829037.949464:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829037.949473:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327304448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829037.949486:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829037.949493:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3195)
00000800:00000200:1.1:1787829037.949980:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829037.949984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829037.950364:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829037.950367:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829037.950774:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829037.950778:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000040:3.0:1787829038.652737:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297858732, at 4297858752
00000100:00000040:2.0:1787829038.652739:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297858732, at 4297858752
00000100:00000010:3.0:1787829038.652754:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e635a0.
00000100:00000010:2.0:1787829038.652761:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ea0.
00000020:00000001:2.0:1787829038.780853:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829038.780857:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829038.780860:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829039.804858:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829039.804862:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829039.804864:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829041.852876:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829041.852880:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829041.852883:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829042.876911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829042.876915:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829042.876918:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787829043.069490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829043.069501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829043.070366:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829043.070368:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829043.070621:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829043.070623:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829043.070844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829043.070846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787829043.580762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829043.580770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829043.580773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829043.580779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829043.580783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829043.580788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829043.580792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829043.580795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829043.580799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829043.772740:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297863852, at 4297863872
00000100:00000040:3.0:1787829043.772740:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297863853, at 4297863872
00000100:00000010:2.0:1787829043.772757:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000100:00000010:3.0:1787829043.772762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000020:00000001:2.0:1787829044.924875:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829044.924881:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829044.924886:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829045.948845:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829045.948849:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829045.948853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829047.996855:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829047.996860:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829047.996865:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829048.188745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829048.188760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829048.188764:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829048.188768:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829048.188773:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829048.188782:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3195
00000400:00000200:2.0:1787829048.188793:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829048.188803:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829048.188808:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829048.188813:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.188817:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.188819:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.188823:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829048.188828:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829048.188831:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829048.188835:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829048.188838:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829048.188842:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829048.188845:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829048.188849:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.188851:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.188854:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.188856:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829048.188860:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829048.188865:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829048.188868:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829048.188872:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829048.188877:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829048.188881:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.188884:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829048.188887:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829048.188898:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829048.188902:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829048.188908:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829048.188910:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.188913:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829048.188917:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829048.188922:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829048.188931:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829048.188940:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829048.188953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327305088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829048.188966:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829048.188972:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829048.188978:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327305088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829048.189005:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829048.189009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829048.189012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829048.189016:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829048.189018:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829048.189022:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829048.189026:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3195
00000400:00000200:2.0:1787829048.189032:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829048.189041:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829048.189050:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829048.189072:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189083:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.189096:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189104:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829048.189112:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829048.189117:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829048.189123:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829048.189132:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829048.189140:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829048.189152:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829048.189164:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189177:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.189187:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189197:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829048.189207:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829048.189216:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189223:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829048.189229:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829048.189239:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829048.189244:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.189253:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829048.189264:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829048.189274:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.189279:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829048.189285:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829048.189291:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829048.189303:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829048.189310:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.189321:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829048.189326:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829048.189331:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829048.189342:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787829048.189351:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829048.189361:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327305216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829048.189392:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829048.189400:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829048.189410:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327305216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829048.189437:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829048.189446:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1787829048.189462:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829048.189468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829048.189482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829048.189487:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829048.189493:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829048.189501:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829048.189514:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3195
00000400:00000200:2.0:1787829048.189524:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829048.189539:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829048.189550:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829048.189559:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189568:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.189576:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189581:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829048.189588:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829048.189592:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829048.189599:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829048.189604:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829048.189614:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829048.189626:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829048.189639:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189653:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.189708:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189715:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829048.189725:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829048.189735:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829048.189743:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829048.189747:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829048.189754:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829048.189763:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.189770:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829048.189774:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829048.189784:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.189792:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829048.189807:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829048.189815:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829048.189825:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829048.189831:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.189840:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829048.189846:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829048.189851:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829048.189866:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829048.189879:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829048.189894:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327305344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829048.189958:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829048.189968:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829048.189985:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327305344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829048.190019:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829048.190032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829048.190046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829048.190051:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829048.190056:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829048.190075:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829048.190084:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3195
00000400:00000200:2.0:1787829048.190099:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829048.190114:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829048.190127:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829048.190140:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.190150:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.190160:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.190170:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829048.190179:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829048.190183:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829048.190188:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829048.190196:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829048.190204:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829048.190220:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829048.190235:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.190245:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829048.190262:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829048.190271:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829048.190282:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829048.190297:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829048.190307:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829048.190314:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829048.190321:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829048.190335:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.190345:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829048.190349:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829048.190356:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.190363:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829048.190372:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787829048.190382:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829048.190393:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829048.190398:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829048.190404:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829048.190408:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829048.190414:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829048.190415:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829048.190422:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
02000000:00000010:2.0:1787829048.190428:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829048.190436:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829048.190451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327305472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829048.190488:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829048.190498:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829048.190514:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327305472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829048.190587:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829048.190600:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3205)
00000800:00000200:0.1:1787829048.190820:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829048.190822:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829048.191100:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829048.191102:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1787829048.892760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297868972, at 4297868992
00000100:00000040:3.0:1787829048.892760:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297868972, at 4297868992
00000100:00000010:3.0:1787829048.892773:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000010:2.0:1787829048.892775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000020:00000001:2.0:1787829049.020832:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829049.020836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829049.020839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829051.068822:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829051.068825:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829051.068828:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829052.092835:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829052.092840:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829052.092843:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829053.309517:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829053.309525:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829053.310060:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829053.310062:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829053.310287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829053.310289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829053.310556:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829053.310559:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787829054.012779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297874093, at 4297874112
00000100:00000040:3.0:1787829054.012779:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297874093, at 4297874112
00000100:00000010:2.0:1787829054.012797:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63de0.
00000100:00000010:3.0:1787829054.012801:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63420.
00000020:00000001:2.0:1787829054.140851:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829054.140856:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829054.140861:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829055.164823:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829055.164826:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829055.164830:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829057.212864:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829057.212867:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829057.212870:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829058.236831:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829058.236834:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829058.236836:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829058.429370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829058.429374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829058.430037:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829058.430039:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829058.430282:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829058.430284:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829058.430519:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829058.430520:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787829059.132727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297879212, at 4297879232
00000100:00000010:2.0:1787829059.132744:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e631e0.
00000100:00000040:3.0:1787829059.132752:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297879212, at 4297879232
00000100:00000010:3.0:1787829059.132765:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63840.
00000020:00000001:2.0:1787829060.284900:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829060.284904:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829060.284909:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829061.308882:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829061.308885:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829061.308888:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829063.356833:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829063.356837:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829063.356840:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829063.549596:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829063.549621:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829063.550441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829063.550445:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829063.550735:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829063.550737:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829063.551068:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829063.551070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:2.0:1787829064.252731:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297884333, at 4297884352
00000100:00000040:3.0:1787829064.252731:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297884333, at 4297884352
00000100:00000010:3.0:1787829064.252744:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63060.
00000100:00000010:2.0:1787829064.252749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63240.
00000020:00000001:2.0:1787829064.380833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829064.380836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829064.380839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829066.108766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829066.108773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829066.108791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829066.108797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829066.108800:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829066.108804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829066.108807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829066.108810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829066.108814:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829066.428850:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829066.428854:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829066.428858:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829067.452878:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829067.452883:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829067.452887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829068.669328:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829068.669334:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829068.669852:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829068.669855:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829068.670083:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829068.670085:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829068.670267:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829068.670269:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787829069.372755:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297889452, at 4297889472
00000100:00000040:3.0:1787829069.372755:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297889452, at 4297889472
00000100:00000010:3.0:1787829069.372770:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ae0.
00000100:00000010:2.0:1787829069.372773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f60.
00000020:00000001:2.0:1787829069.500944:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829069.500962:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829069.500965:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829070.525012:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829070.525018:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829070.525022:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829072.573000:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829072.573005:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829072.573009:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829073.276924:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829073.276949:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829073.276952:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829073.276961:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829073.276966:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829073.276971:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829073.276974:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829073.276978:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829073.276982:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829073.596911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829073.596917:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829073.596922:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829073.788780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829073.788797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829073.788800:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829073.788806:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829073.788812:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829073.788823:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3221
00000400:00000200:2.0:1787829073.788835:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829073.788845:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829073.788850:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829073.788854:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.788858:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.788861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.788864:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829073.788870:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829073.788872:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829073.788876:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829073.788895:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829073.788899:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829073.788902:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829073.788906:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.788908:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.788911:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.788913:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829073.788918:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829073.788922:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829073.788926:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829073.788930:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829073.788935:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829073.788938:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.788942:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829073.788945:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829073.788958:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829073.788963:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829073.788968:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829073.788970:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.788974:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829073.788978:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829073.788983:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829073.788995:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829073.789006:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829073.789019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327307648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829073.789042:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829073.789048:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829073.789054:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327307648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829073.789080:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829073.789085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829073.789089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829073.789091:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829073.789092:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829073.789096:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829073.789100:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3221
00000400:00000200:2.0:1787829073.789104:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829073.789107:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829073.789111:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829073.789116:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789119:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.789123:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789126:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829073.789130:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829073.789132:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829073.789134:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829073.789137:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829073.789140:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829073.789143:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829073.789149:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789156:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.789166:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789173:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829073.789200:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829073.789218:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789237:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829073.789245:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829073.789256:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829073.789266:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.789274:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829073.789286:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829073.789326:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.789329:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829073.789341:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787829073.789350:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829073.789362:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829073.789381:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.789387:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829073.789392:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829073.789395:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829073.789406:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829073.789420:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829073.789439:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327307776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829073.789475:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829073.789481:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829073.789496:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327307776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829073.789525:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829073.789533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829073.789547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829073.789555:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829073.789564:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829073.789578:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829073.789589:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3221
00000400:00000200:2.0:1787829073.789614:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829073.789642:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829073.789652:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:1.1:1787829073.789665:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829073.789670:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787829073.789700:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789704:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.789707:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789709:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829073.789714:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829073.789716:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829073.789718:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829073.789724:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829073.789727:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829073.789731:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829073.789736:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789740:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.789743:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789746:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829073.789751:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829073.789764:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829073.789792:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829073.789807:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829073.789817:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829073.789824:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.789827:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829073.789849:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829073.789857:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.789864:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829073.789873:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787829073.789899:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829073.789923:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829073.789926:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.789933:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829073.789941:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829073.789950:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829073.789965:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829073.789990:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829073.790007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327307904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829073.790030:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829073.790040:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829073.790053:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327307904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829073.790117:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829073.790127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829073.790139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829073.790149:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829073.790157:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829073.790168:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829073.790182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3221
00000400:00000200:2.0:1787829073.790209:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829073.790237:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829073.790246:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829073.790262:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.790272:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.790284:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.790305:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829073.790322:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829073.790330:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829073.790353:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829073.790359:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829073.790370:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829073.790384:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829073.790408:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.790418:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829073.790429:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829073.790443:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829073.790461:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829073.790474:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829073.790482:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829073.790493:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829073.790508:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829073.790518:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.790531:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829073.790537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829073.790550:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.790555:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829073.790571:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787829073.790580:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829073.790594:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829073.790603:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829073.790613:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829073.790620:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829073.790628:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829073.790661:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829073.790673:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829073.790689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327308032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829073.790738:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829073.790742:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829073.790782:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327308032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829073.790852:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829073.790865:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3231)
00000800:00000200:1.1:1787829073.790966:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829073.790970:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829073.791334:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829073.791337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829073.791934:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829073.791939:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787829074.300761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829074.300786:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829074.300789:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829074.300794:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829074.300797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829074.300801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829074.300803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829074.300806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829074.300808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829074.492747:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297894573, at 4297894592
00000100:00000040:2.0:1787829074.492762:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297894573, at 4297894592
00000100:00000010:3.0:1787829074.492765:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63360.
00000100:00000010:2.0:1787829074.492778:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e638a0.
00000100:00000001:2.0:1787829075.324775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829075.324783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829075.324786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829075.324791:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829075.324794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829075.324797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829075.324800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829075.324803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829075.324805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829075.644831:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829075.644837:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829075.644843:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829076.348764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829076.348771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829076.348772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829076.348776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829076.348778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829076.348781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829076.348783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829076.348785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829076.348787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829076.668885:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829076.668888:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829076.668891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829077.372752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829077.372759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829077.372760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829077.372764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829077.372767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829077.372769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829077.372772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829077.372774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829077.372776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.396728:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829078.396734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829078.396735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829078.396740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829078.396742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829078.396745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829078.396747:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829078.396749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829078.396751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829078.716965:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829078.716973:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829078.716980:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829078.908791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829078.908806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829078.908810:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829078.908815:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829078.908822:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829078.908831:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3226
00000400:00000200:2.0:1787829078.908842:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829078.908868:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829078.908873:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829078.908878:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.908882:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.908886:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.908889:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829078.908895:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829078.908898:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829078.908902:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829078.908906:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829078.908910:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829078.908914:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829078.908919:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.908922:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.908927:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.908930:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829078.908934:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829078.908939:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829078.908942:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829078.908945:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829078.908949:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829078.908952:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.908955:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829078.908958:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829078.908968:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787829078.908972:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829078.908977:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829078.908979:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.908983:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829078.908987:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829078.908992:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829078.909002:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829078.909009:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829078.909021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327308160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829078.909034:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829078.909039:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829078.909045:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327308160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829078.909071:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829078.909075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829078.909080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829078.909082:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829078.909084:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829078.909087:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829078.909090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3226
00000400:00000200:2.0:1787829078.909094:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829078.909098:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829078.909100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829078.909104:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909107:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.909110:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909113:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829078.909116:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829078.909119:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829078.909122:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829078.909125:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829078.909128:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829078.909131:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829078.909135:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909139:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.909142:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909145:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829078.909149:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829078.909157:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909168:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829078.909173:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829078.909182:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829078.909190:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909198:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829078.909206:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829078.909212:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909217:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829078.909227:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787829078.909236:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829078.909245:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829078.909247:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909249:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829078.909251:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829078.909254:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829078.909258:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829078.909262:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829078.909276:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327308288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829078.909301:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829078.909305:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829078.909312:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327308288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829078.909333:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829078.909338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829078.909342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829078.909345:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829078.909350:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829078.909358:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829078.909367:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3226
00000400:00000200:2.0:1787829078.909382:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829078.909393:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829078.909401:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829078.909408:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909418:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.909421:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909425:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829078.909428:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829078.909430:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829078.909432:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829078.909434:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829078.909436:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829078.909439:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829078.909452:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909458:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.909467:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909471:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829078.909477:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829078.909489:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909499:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829078.909505:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829078.909513:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829078.909523:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909529:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829078.909536:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829078.909546:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909550:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829078.909560:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787829078.909568:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829078.909574:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829078.909576:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909582:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829078.909588:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829078.909591:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829078.909599:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829078.909607:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829078.909623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327308416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829078.909649:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829078.909705:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829078.909715:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327308416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829078.909747:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829078.909752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829078.909764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829078.909771:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829078.909777:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829078.909784:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829078.909793:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3226
00000400:00000200:2.0:1787829078.909802:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829078.909814:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829078.909819:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829078.909829:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909835:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.909844:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829078.909870:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829078.909872:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829078.909874:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829078.909877:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:1.1:1787829078.909885:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787829078.909886:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829078.909893:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:1.1:1787829078.909900:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1787829078.909903:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909912:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829078.909921:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909927:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829078.909936:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829078.909941:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829078.909948:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829078.909950:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829078.909954:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829078.909959:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909967:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829078.909973:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829078.909980:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.909990:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829078.910002:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787829078.910009:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829078.910018:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829078.910021:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829078.910030:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829078.910034:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829078.910039:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829078.910051:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829078.910060:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829078.910076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327308544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829078.910100:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829078.910104:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829078.910115:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327308544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829078.910143:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829078.910150:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3236)
00000800:00000200:1.1:1787829078.910624:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829078.910627:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829078.911059:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829078.911063:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829078.911463:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829078.911466:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787829079.420755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829079.420764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829079.420767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829079.420773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829079.420777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829079.420781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829079.420784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829079.420786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829079.420789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829079.612760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297899693, at 4297899712
00000100:00000040:3.0:1787829079.612760:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297899693, at 4297899712
00000100:00000010:2.0:1787829079.612775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63720.
00000100:00000010:3.0:1787829079.612777:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e634e0.
00000020:00000001:2.0:1787829079.740805:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829079.740809:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829079.740811:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829080.444771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829080.444780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829080.444783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829080.444790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829080.444796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829080.444801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829080.444804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829080.444807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829080.444811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829081.468782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829081.468792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829081.468795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829081.468801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829081.468806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829081.468811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829081.468815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829081.468819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829081.468823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829081.788856:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829081.788860:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829081.788863:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829082.492732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829082.492741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829082.492743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829082.492748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829082.492750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829082.492753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829082.492755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829082.492757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829082.492759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829082.812906:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829082.812910:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829082.812912:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829083.516742:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829083.516750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829083.516752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829083.516758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829083.516762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829083.516766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829083.516769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829083.516772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829083.516783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829084.028800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829084.028826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829084.028829:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829084.028833:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829084.028838:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829084.028858:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3231
00000400:00000200:3.0:1787829084.028872:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829084.028878:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829084.028881:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829084.028884:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.028886:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.028888:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.028890:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829084.028894:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829084.028896:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829084.028898:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829084.028902:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829084.028905:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829084.028906:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829084.028909:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.028910:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.028912:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.028913:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829084.028915:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829084.028917:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829084.028919:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829084.028921:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829084.028925:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829084.028927:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.028929:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829084.028931:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829084.028941:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829084.028944:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829084.028947:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829084.028949:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.028952:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829084.028955:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829084.028959:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829084.028967:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829084.028972:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829084.028981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327308672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829084.029005:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829084.029010:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829084.029014:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327308672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829084.029038:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829084.029040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829084.029044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829084.029072:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829084.029073:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829084.029076:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829084.029078:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3231
00000400:00000200:3.0:1787829084.029080:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829084.029086:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829084.029088:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829084.029090:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029092:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.029094:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029096:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829084.029099:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829084.029100:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829084.029102:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829084.029105:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829084.029108:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829084.029113:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829084.029121:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029127:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.029135:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029147:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829084.029155:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829084.029162:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029166:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829084.029170:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829084.029178:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829084.029182:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029186:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829084.029191:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829084.029197:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029200:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829084.029204:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829084.029209:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829084.029212:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829084.029213:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029214:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829084.029215:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829084.029217:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829084.029222:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829084.029228:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829084.029246:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327308800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829084.029266:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829084.029272:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829084.029283:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327308800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829084.029304:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829084.029308:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829084.029310:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829084.029312:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829084.029313:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829084.029318:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829084.029323:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3231
00000400:00000200:3.0:1787829084.029331:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829084.029337:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829084.029340:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829084.029346:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029350:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.029355:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029360:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829084.029365:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829084.029368:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829084.029372:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829084.029376:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829084.029379:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829084.029396:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829084.029401:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029406:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.029412:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029417:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829084.029419:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829084.029426:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029430:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829084.029431:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829084.029436:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829084.029439:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029443:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829084.029448:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829084.029453:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029456:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829084.029460:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000800:00000200:1.1:1787829084.029461:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787829084.029463:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:1.1:1787829084.029484:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
02000000:00000001:3.0:1787829084.029485:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829084.029487:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029491:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829084.029493:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829084.029495:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829084.029500:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829084.029505:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829084.029513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327308928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829084.029544:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829084.029548:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829084.029569:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327308928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829084.029598:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829084.029602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829084.029606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829084.029607:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829084.029608:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829084.029611:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829084.029617:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3231
00000400:00000200:3.0:1787829084.029627:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829084.029632:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829084.029637:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829084.029639:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029642:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.029647:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029650:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829084.029652:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829084.029653:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829084.029692:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829084.029697:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829084.029704:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829084.029711:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829084.029721:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029726:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829084.029732:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029737:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829084.029742:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829084.029764:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829084.029767:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829084.029770:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829084.029775:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829084.029779:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029784:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829084.029788:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829084.029794:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029798:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829084.029805:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829084.029822:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829084.029827:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829084.029830:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829084.029835:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829084.029838:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829084.029843:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829084.029878:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829084.029883:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829084.029894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327309056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829084.029915:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829084.029919:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829084.029929:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327309056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829084.029952:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829084.029962:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3241)
00000800:00000200:1.1:1787829084.030185:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829084.030187:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829084.030427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829084.030429:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829084.030746:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829084.030748:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1787829084.540795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829084.540803:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829084.540805:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829084.540824:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829084.540828:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829084.540832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829084.540834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829084.540838:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829084.540842:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829084.732770:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297904812, at 4297904832
00000100:00000010:2.0:1787829084.732785:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e636c0.
00000100:00000040:3.0:1787829084.732786:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297904812, at 4297904832
00000100:00000010:3.0:1787829084.732803:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63c60.
00000020:00000001:2.0:1787829084.860889:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829084.860894:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829084.860899:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829085.564735:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829085.564741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829085.564743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829085.564747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829085.564750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829085.564753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829085.564755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829085.564757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829085.564759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829085.884897:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829085.884903:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829085.884907:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829086.588755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829086.588762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829086.588764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829086.588769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829086.588773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829086.588778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829086.588781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829086.588784:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829086.588787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829087.933006:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829087.933011:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829087.933017:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829088.956824:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829088.956830:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829088.956834:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829089.148748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829089.148759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829089.148763:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829089.148767:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829089.148772:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829089.148793:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3236
00000400:00000200:3.0:1787829089.148804:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829089.148810:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829089.148825:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829089.148828:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.148831:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.148833:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.148835:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829089.148839:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829089.148840:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829089.148843:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829089.148846:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829089.148848:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829089.148850:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829089.148852:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.148853:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.148860:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.148861:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829089.148864:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829089.148867:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829089.148870:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829089.148872:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829089.148875:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829089.148878:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.148880:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829089.148882:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829089.148892:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829089.148895:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829089.148900:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829089.148902:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.148905:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829089.148908:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829089.148913:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829089.148921:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829089.148929:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829089.148942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327309184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829089.148954:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829089.148960:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829089.148966:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327309184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829089.149005:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829089.149016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829089.149024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829089.149027:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829089.149029:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829089.149033:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829089.149048:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3236
00000400:00000200:3.0:1787829089.149053:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829089.149056:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829089.149059:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829089.149066:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149070:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.149075:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149081:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829089.149084:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829089.149087:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829089.149090:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829089.149095:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829089.149110:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829089.149116:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829089.149124:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149130:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.149134:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149139:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829089.149146:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829089.149153:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149161:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829089.149164:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829089.149184:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829089.149189:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149194:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829089.149202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829089.149210:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149215:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829089.149221:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829089.149225:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829089.149230:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829089.149233:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149236:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829089.149238:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829089.149241:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829089.149249:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829089.149257:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829089.149269:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327309312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829089.149290:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829089.149296:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829089.149304:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327309312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829089.149333:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829089.149337:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829089.149343:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829089.149347:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829089.149351:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829089.149359:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829089.149367:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3236
00000400:00000200:3.0:1787829089.149388:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829089.149398:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829089.149404:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829089.149412:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829089.149418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829089.149419:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.149425:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149428:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829089.149431:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829089.149433:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829089.149434:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829089.149437:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829089.149441:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829089.149448:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829089.149454:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149460:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.149465:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149469:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829089.149474:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829089.149480:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149487:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829089.149494:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829089.149502:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829089.149509:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149514:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829089.149519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829089.149524:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149528:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829089.149538:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829089.149543:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829089.149550:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829089.149553:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149557:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829089.149561:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829089.149573:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829089.149582:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829089.149589:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829089.149600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327309440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829089.149623:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829089.149627:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829089.149633:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327309440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829089.149642:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829089.149648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829089.149676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829089.149678:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829089.149679:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829089.149682:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829089.149685:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3236
00000400:00000200:3.0:1787829089.149689:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829089.149700:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829089.149702:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829089.149706:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149707:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.149710:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149712:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829089.149716:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829089.149718:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829089.149722:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829089.149725:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829089.149730:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829089.149734:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829089.149738:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149743:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829089.149748:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149753:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829089.149757:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829089.149763:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829089.149767:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829089.149770:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829089.149787:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829089.149790:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149792:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829089.149795:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829089.149799:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149801:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829089.149808:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829089.149813:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829089.149817:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829089.149819:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829089.149822:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829089.149823:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829089.149833:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829089.149838:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829089.149844:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829089.149853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327309568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829089.149878:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829089.149883:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829089.149897:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327309568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829089.149914:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829089.149915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829089.149921:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3246)
00000800:00000200:1.1:1787829089.150125:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829089.150287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829089.852744:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297909932, at 4297909952
00000100:00000040:2.0:1787829089.852760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297909932, at 4297909952
00000100:00000010:3.0:1787829089.852760:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63660.
00000100:00000010:2.0:1787829089.852786:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a80.
00000020:00000001:2.0:1787829091.004861:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829091.004865:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829091.004867:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829092.028946:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829092.028953:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829092.028958:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829092.220997:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829092.221033:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829092.221041:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829092.221060:0:2334:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829092.221065:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829092.221069:0:2334:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829092.221073:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829092.221076:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955d00.
00000400:00000200:2.0:1787829092.221080:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829092.221084:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829092.221090:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f984c8
00000400:00000010:2.0:1787829092.221092:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f984c8.
00000100:00000001:2.0:1787829092.221097:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829092.221104:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0000 x1874671327309184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221128:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829092.221132:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327309184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221138:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829092.221140:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829092.221148:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829092.221150:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:2.0:1787829092.221168:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829092.221170:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829092.221173:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98440
00000400:00000010:2.0:1787829092.221174:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98440.
00000100:00000001:2.0:1787829092.221177:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829092.221180:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed2d00 x1874671327309312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221187:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829092.221190:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327309312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221198:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829092.221200:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829092.221205:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829092.221211:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955900.
00000400:00000200:2.0:1787829092.221216:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829092.221223:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829092.221227:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98f68
00000400:00000010:2.0:1787829092.221231:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98f68.
00000100:00000001:2.0:1787829092.221235:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829092.221243:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed30c0 x1874671327309440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221256:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829092.221262:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327309440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221274:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829092.221279:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829092.221290:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829092.221294:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:2.0:1787829092.221300:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829092.221311:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829092.221316:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f986e8
00000400:00000010:2.0:1787829092.221321:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f986e8.
00000100:00000001:2.0:1787829092.221327:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829092.221335:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0780 x1874671327309568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221349:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829092.221358:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327309568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829092.221390:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829092.221394:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:2.0:1787829094.076867:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829094.076871:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829094.076874:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829094.268751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829094.268763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829094.268766:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829094.268769:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829094.268773:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829094.268779:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3241
00000400:00000200:3.0:1787829094.268786:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829094.268791:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829094.268794:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829094.268796:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268798:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.268800:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268802:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829094.268805:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829094.268806:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829094.268808:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829094.268811:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829094.268813:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829094.268815:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829094.268818:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268819:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.268821:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268822:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829094.268824:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829094.268827:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268828:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829094.268831:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829094.268833:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829094.268835:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.268837:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829094.268838:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829094.268844:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829094.268846:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829094.268849:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829094.268850:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.268852:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829094.268855:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829094.268858:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829094.268866:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829094.268873:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829094.268882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327309696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829094.268893:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829094.268897:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829094.268903:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327309696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829094.268932:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829094.268936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829094.268940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829094.268941:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829094.268944:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829094.268947:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829094.268949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3241
00000400:00000200:3.0:1787829094.268954:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829094.268958:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829094.268961:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829094.268965:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268968:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.268971:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.268973:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829094.268977:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829094.268980:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829094.268982:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829094.268986:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829094.268989:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829094.268991:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829094.269005:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269010:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.269015:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269019:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829094.269024:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829094.269033:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269039:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829094.269043:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829094.269051:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829094.269058:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269062:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829094.269067:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829094.269071:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269073:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829094.269080:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829094.269082:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829094.269085:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829094.269088:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269090:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829094.269091:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829094.269093:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829094.269098:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829094.269103:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829094.269110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327309824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829094.269121:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829094.269126:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829094.269134:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327309824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829094.269153:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829094.269160:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829094.269166:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829094.269171:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829094.269173:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829094.269177:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829094.269184:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3241
00000400:00000200:3.0:1787829094.269200:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829094.269210:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829094.269217:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829094.269226:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269231:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.269235:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269240:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829094.269246:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829094.269250:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829094.269253:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829094.269257:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829094.269260:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829094.269263:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829094.269271:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269276:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.269282:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269285:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829094.269290:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829094.269297:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:1.1:1787829094.269298:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829094.269303:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829094.269305:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
02000000:00000001:3.0:1787829094.269308:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829094.269313:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829094.269317:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269322:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829094.269326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829094.269330:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269335:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829094.269341:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829094.269346:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829094.269352:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829094.269356:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269361:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829094.269377:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829094.269382:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829094.269396:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829094.269402:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829094.269414:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327309952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829094.269432:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829094.269436:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829094.269443:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327309952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829094.269458:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829094.269465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829094.269472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829094.269477:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829094.269479:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829094.269486:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829094.269493:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3241
00000400:00000200:3.0:1787829094.269501:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829094.269506:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829094.269508:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829094.269602:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269608:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.269615:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269622:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829094.269627:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829094.269629:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829094.269633:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829094.269636:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829094.269638:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829094.269641:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829094.269647:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269652:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829094.269680:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269683:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829094.269686:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829094.269689:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829094.269693:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829094.269695:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829094.269700:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829094.269705:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269710:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829094.269714:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829094.269721:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269725:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829094.269733:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829094.269754:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829094.269761:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829094.269765:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829094.269769:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829094.269772:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829094.269776:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829094.269784:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829094.269790:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829094.269799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327310080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829094.269815:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829094.269830:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829094.269837:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327310080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829094.269855:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829094.269861:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3252)
00000800:00000200:1.1:1787829094.270016:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829094.270019:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787829094.270261:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829094.270263:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787829094.270509:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829094.270511:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1787829094.972717:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297915052, at 4297915072
00000100:00000040:2.0:1787829094.972726:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297915052, at 4297915072
00000100:00000010:3.0:1787829094.972746:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e633c0.
00000100:00000010:2.0:1787829094.972749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63600.
00000020:00000001:2.0:1787829095.100806:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829095.100810:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829095.100813:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829097.148818:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829097.148825:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829097.148828:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829098.172869:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829098.172873:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829098.172876:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829099.388783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829099.388794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829099.388796:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829099.388800:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829099.388805:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829099.388812:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3247
00000400:00000200:2.0:1787829099.388821:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829099.388828:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829099.388831:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829099.388835:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.388838:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.388841:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.388843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829099.388847:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829099.388850:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829099.388852:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829099.388856:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829099.388859:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829099.388862:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829099.388865:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.388868:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.388870:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.388872:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829099.388876:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829099.388880:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829099.388884:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829099.388886:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829099.388890:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829099.388893:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.388896:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829099.388899:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829099.388908:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829099.388912:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829099.388916:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829099.388919:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.388922:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829099.388925:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829099.388929:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829099.388936:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829099.388943:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829099.388952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327310208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829099.388963:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829099.388968:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829099.388973:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327310208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829099.389004:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829099.389008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829099.389012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829099.389013:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829099.389015:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829099.389019:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829099.389023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3247
00000400:00000200:2.0:1787829099.389028:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829099.389032:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829099.389036:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829099.389044:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389050:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.389056:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389063:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829099.389071:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829099.389075:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829099.389080:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829099.389084:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829099.389089:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829099.389095:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829099.389112:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389117:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.389122:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389127:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829099.389132:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829099.389136:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389142:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829099.389146:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829099.389152:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829099.389156:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389165:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829099.389172:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829099.389178:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389185:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829099.389192:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787829099.389197:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829099.389202:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829099.389205:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389210:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829099.389213:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829099.389216:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829099.389224:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829099.389231:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829099.389241:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327310336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829099.389253:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829099.389255:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829099.389263:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327310336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829099.389277:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829099.389282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829099.389288:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829099.389292:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829099.389294:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829099.389300:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829099.389308:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3247
00000400:00000200:2.0:1787829099.389318:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829099.389327:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829099.389331:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829099.389340:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389347:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.389351:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389356:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829099.389361:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829099.389363:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829099.389368:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829099.389372:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829099.389377:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829099.389382:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829099.389391:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389397:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.389405:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829099.389409:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829099.389411:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:1.1:1787829099.389413:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000020:00000040:2.0:1787829099.389426:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829099.389435:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389441:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829099.389447:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829099.389453:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829099.389460:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389465:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829099.389470:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829099.389477:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389482:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829099.389489:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829099.389498:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829099.389505:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829099.389509:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389516:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829099.389520:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829099.389524:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829099.389532:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829099.389538:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829099.389550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327310464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829099.389573:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829099.389578:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829099.389590:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327310464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829099.389611:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829099.389618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829099.389624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829099.389629:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829099.389634:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829099.389641:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829099.389648:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3247
00000400:00000200:2.0:1787829099.389694:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829099.389717:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829099.389724:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829099.389732:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389739:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.389745:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389750:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829099.389756:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829099.389759:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829099.389763:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829099.389766:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829099.389771:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829099.389776:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829099.389784:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389790:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829099.389796:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389802:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829099.389808:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829099.389815:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829099.389821:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829099.389825:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829099.389829:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829099.389835:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389840:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829099.389844:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829099.389850:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389855:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829099.389862:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829099.389868:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829099.389873:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829099.389877:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829099.389882:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829099.389886:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829099.389890:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829099.389897:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829099.389903:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829099.389911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327310592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829099.389927:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829099.389932:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829099.389939:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327310592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829099.389963:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829099.389971:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3257)
00000800:00000200:1.1:1787829099.390074:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829099.390077:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829099.390377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829099.390380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829099.390737:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829099.390739:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787829100.092765:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297920172, at 4297920192
00000100:00000040:3.0:1787829100.092765:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297920172, at 4297920192
00000100:00000010:3.0:1787829100.092781:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63900.
00000100:00000010:2.0:1787829100.092782:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e632a0.
00000020:00000001:2.0:1787829100.220839:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829100.220843:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829100.220847:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829101.244884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829101.244887:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829101.244890:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829101.948769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829101.948774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829101.948776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829101.948780:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829101.948783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829101.948786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829101.948788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829101.948790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829101.948792:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829102.972771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829102.972777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829102.972779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829102.972783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829102.972786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829102.972789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829102.972791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829102.972792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829102.972795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829103.292835:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829103.292838:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829103.292841:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829103.996765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829103.996771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829103.996773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829103.996777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829103.996779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829103.996782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829103.996784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829103.996787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829103.996789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829104.316837:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829104.316841:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829104.316843:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829104.509490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829104.509501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829104.510166:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829104.510169:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829104.510398:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829104.510400:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829104.510632:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829104.510649:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829105.020767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829105.020773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829105.020775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829105.020778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829105.020781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829105.020784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829105.020786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829105.020788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829105.020791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829105.212743:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297925293, at 4297925312
00000100:00000010:3.0:1787829105.212760:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e632a0.
00000100:00000040:2.0:1787829105.212772:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297925293, at 4297925312
00000100:00000010:2.0:1787829105.212786:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63900.
00000100:00000001:2.0:1787829106.044768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829106.044774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829106.044776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829106.044779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829106.044782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829106.044785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829106.044787:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829106.044789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829106.044791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829106.364834:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829106.364840:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829106.364843:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829107.068725:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829107.068731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829107.068732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829107.068736:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829107.068739:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829107.068741:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829107.068743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829107.068745:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829107.068747:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829107.388831:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829107.388836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829107.388840:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829108.092781:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829108.092788:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829108.092790:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829108.092793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829108.092796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829108.092799:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829108.092801:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829108.092803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829108.092805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829109.116764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829109.116769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829109.116771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829109.116774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829109.116779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829109.116782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829109.116785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829109.116789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829109.116793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829109.436938:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829109.436943:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829109.436947:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829109.628785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829109.628798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829109.628801:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829109.628805:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829109.628810:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829109.628818:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3257
00000400:00000200:3.0:1787829109.628827:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829109.628834:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829109.628839:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829109.628842:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.628847:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.628851:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.628854:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829109.628860:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829109.628863:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829109.628866:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829109.628871:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829109.628874:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829109.628877:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829109.628881:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.628884:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.628887:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.628889:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829109.628893:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829109.628897:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829109.628900:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829109.628903:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829109.628908:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829109.628912:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.628914:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829109.628917:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829109.628930:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829109.628935:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829109.628940:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829109.628943:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.628947:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829109.628950:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829109.628955:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829109.628966:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829109.628975:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829109.628986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327311232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829109.628998:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829109.629004:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829109.629010:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327311232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829109.629036:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829109.629040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829109.629044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829109.629046:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829109.629048:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829109.629052:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829109.629054:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3257
00000400:00000200:3.0:1787829109.629059:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829109.629062:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829109.629065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829109.629074:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629077:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.629088:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629091:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829109.629094:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829109.629096:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829109.629098:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829109.629102:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829109.629105:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829109.629108:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829109.629112:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629114:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.629118:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629123:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829109.629127:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829109.629133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629139:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829109.629143:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829109.629152:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829109.629158:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629162:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829109.629170:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829109.629177:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629183:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829109.629190:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829109.629197:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829109.629202:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829109.629206:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629210:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829109.629213:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829109.629218:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829109.629225:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829109.629233:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829109.629243:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327311360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829109.629269:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829109.629275:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829109.629285:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327311360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829109.629311:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829109.629315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829109.629320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829109.629328:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829109.629332:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829109.629340:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829109.629346:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3257
00000400:00000200:3.0:1787829109.629355:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829109.629363:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829109.629368:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829109.629383:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629388:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.629393:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629397:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829109.629400:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829109.629402:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829109.629406:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829109.629411:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829109.629417:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829109.629423:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829109.629432:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629438:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.629442:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629447:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829109.629453:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829109.629461:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:1.1:1787829109.629463:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829109.629469:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829109.629470:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
02000000:00000001:3.0:1787829109.629477:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829109.629483:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829109.629488:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629493:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829109.629499:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829109.629506:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629511:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829109.629519:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829109.629526:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829109.629532:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829109.629535:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629540:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829109.629544:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829109.629548:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829109.629556:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829109.629564:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829109.629576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327311488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829109.629587:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829109.629592:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829109.629599:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327311488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829109.629620:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829109.629625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829109.629648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829109.629651:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829109.629688:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829109.629695:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829109.629702:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3257
00000400:00000200:3.0:1787829109.629712:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829109.629721:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829109.629727:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829109.629736:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629741:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.629748:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629754:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829109.629760:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829109.629763:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829109.629768:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829109.629772:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829109.629779:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829109.629785:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829109.629795:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629803:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829109.629811:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629818:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829109.629826:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829109.629836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829109.629843:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829109.629849:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829109.629855:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829109.629861:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629866:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829109.629871:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829109.629876:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629880:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829109.629884:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829109.629888:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829109.629894:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829109.629898:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829109.629900:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829109.629903:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829109.629907:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829109.629912:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829109.629917:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829109.629925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327311616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829109.629942:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829109.629946:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829109.629954:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327311616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829109.629970:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829109.629977:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3267)
00000800:00000200:1.1:1787829109.630058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829109.630060:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829109.630247:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829109.630248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829109.630606:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829109.630608:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787829110.140807:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829110.140814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829110.140817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829110.140822:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829110.140827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829110.140831:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829110.140834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829110.140837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829110.140841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829110.332707:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297930412, at 4297930432
00000100:00000010:3.0:1787829110.332724:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e633c0.
00000100:00000040:2.0:1787829110.332734:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297930412, at 4297930432
00000100:00000010:2.0:1787829110.332743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63600.
00000020:00000001:2.0:1787829110.460771:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829110.460774:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829110.460777:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829111.164744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829111.164749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829111.164757:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829111.164762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829111.164765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829111.164767:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829111.164769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829111.164771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829111.164774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829112.188773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829112.188779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829112.188781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829112.188785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829112.188788:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829112.188791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829112.188793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829112.188796:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829112.188800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829112.508883:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829112.508888:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829112.508891:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829113.212721:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829113.212727:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829113.212729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829113.212733:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829113.212735:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829113.212738:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829113.212740:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829113.212742:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829113.212744:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829113.532860:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829113.532865:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829113.532869:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829114.236745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829114.236752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829114.236754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829114.236759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829114.236762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829114.236766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829114.236769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829114.236771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829114.236774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829114.748772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829114.748780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829114.748782:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829114.748785:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829114.748789:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829114.748794:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3262
00000400:00000200:3.0:1787829114.748801:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829114.748806:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829114.748808:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829114.748810:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748813:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.748815:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748817:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829114.748819:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829114.748821:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829114.748822:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829114.748825:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829114.748827:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829114.748828:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829114.748832:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748833:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.748834:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748836:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829114.748838:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829114.748841:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748842:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829114.748844:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829114.748846:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829114.748848:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.748850:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829114.748852:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829114.748865:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829114.748868:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829114.748871:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829114.748872:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.748875:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829114.748877:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829114.748879:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829114.748886:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829114.748890:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829114.748897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327311744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829114.748905:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829114.748908:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829114.748912:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327311744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829114.748929:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829114.748931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829114.748933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829114.748934:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829114.748936:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829114.748938:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829114.748940:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3262
00000400:00000200:3.0:1787829114.748943:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829114.748945:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829114.748946:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829114.748948:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748950:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.748951:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748953:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829114.748954:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829114.748955:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829114.748956:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829114.748958:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829114.748960:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829114.748961:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829114.748964:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748965:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.748967:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748968:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829114.748970:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829114.748972:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829114.748974:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829114.748975:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829114.748979:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829114.748982:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.748984:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829114.748990:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829114.748995:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.748996:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829114.748999:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829114.749002:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829114.749005:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829114.749007:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749010:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829114.749011:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829114.749014:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829114.749020:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829114.749024:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829114.749032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327311872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829114.749040:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829114.749044:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829114.749048:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327311872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829114.749057:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829114.749061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829114.749064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829114.749068:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829114.749069:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829114.749072:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829114.749076:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3262
00000400:00000200:3.0:1787829114.749081:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829114.749084:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829114.749087:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829114.749089:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749090:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.749092:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749095:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829114.749099:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829114.749101:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829114.749104:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829114.749106:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829114.749107:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829114.749111:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829114.749116:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749118:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.749120:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749124:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829114.749127:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829114.749129:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749131:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829114.749132:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829114.749135:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829114.749137:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749139:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829114.749140:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829114.749142:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749145:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829114.749151:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829114.749154:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829114.749159:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829114.749161:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749163:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829114.749165:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829114.749167:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829114.749172:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829114.749177:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829114.749184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327312000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829114.749195:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829114.749197:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829114.749205:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327312000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829114.749215:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829114.749218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829114.749222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829114.749224:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829114.749225:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829114.749227:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829114.749229:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3262
00000400:00000200:3.0:1787829114.749235:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829114.749238:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829114.749240:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829114.749244:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749246:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.749251:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749254:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829114.749256:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829114.749258:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829114.749259:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829114.749263:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829114.749265:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829114.749267:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829114.749272:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749274:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829114.749278:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749280:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829114.749284:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829114.749289:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829114.749293:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829114.749296:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829114.749300:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829114.749304:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749309:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829114.749317:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829114.749319:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749323:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829114.749327:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829114.749331:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829114.749337:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829114.749339:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829114.749341:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829114.749343:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829114.749345:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829114.749349:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829114.749353:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829114.749358:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327312128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829114.749368:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829114.749369:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829114.749377:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327312128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829114.749387:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829114.749391:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829114.749393:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3272)
00000800:00000200:1.1:1787829114.749399:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787829114.749909:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829114.749912:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787829114.750221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829114.750223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787829114.750564:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829114.750567:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1787829115.260768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829115.260778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829115.260780:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829115.260786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829115.260791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829115.260796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829115.260799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829115.260803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829115.260806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829115.452820:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297935532, at 4297935552
00000100:00000040:3.0:1787829115.452820:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297935532, at 4297935552
00000100:00000010:2.0:1787829115.452843:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a80.
00000100:00000010:3.0:1787829115.452849:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63660.
00000020:00000001:2.0:1787829115.580846:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829115.580852:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829115.580857:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829116.284803:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829116.284812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829116.284814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829116.284820:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829116.284825:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829116.284828:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829116.284831:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829116.284833:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829116.284837:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829116.604821:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829116.604826:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829116.604829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829117.308754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829117.308762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829117.308765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829117.308771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829117.308775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829117.308778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829117.308780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829117.308782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829117.308785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829118.332759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829118.332768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829118.332770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829118.332778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829118.332782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829118.332785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829118.332786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829118.332788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829118.332790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829118.652851:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829118.652856:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829118.652860:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829119.356758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829119.356764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829119.356766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829119.356769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829119.356772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829119.356775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829119.356777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829119.356779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829119.356781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829119.676845:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829119.676850:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829119.676853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829119.868754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829119.868769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829119.868772:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829119.868778:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829119.868784:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829119.868794:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3267
00000400:00000200:3.0:1787829119.868807:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829119.868815:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829119.868818:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829119.868821:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.868823:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.868826:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.868828:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829119.868831:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829119.868833:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829119.868835:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829119.868839:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829119.868840:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829119.868842:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829119.868844:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.868846:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.868847:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.868849:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829119.868851:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829119.868859:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829119.868861:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829119.868863:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829119.868867:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829119.868869:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.868871:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829119.868874:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829119.868885:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829119.868888:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829119.868891:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829119.868893:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.868896:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829119.868899:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829119.868903:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829119.868911:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829119.868917:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829119.868927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327312256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829119.868936:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829119.868940:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829119.868943:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327312256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829119.868964:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829119.868967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829119.868972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829119.868974:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829119.868975:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829119.868979:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829119.868981:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3267
00000400:00000200:3.0:1787829119.868985:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829119.868988:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829119.868990:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829119.868994:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.868996:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.868998:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869000:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829119.869003:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829119.869004:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829119.869006:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829119.869009:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829119.869011:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829119.869013:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829119.869017:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869019:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.869021:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869024:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829119.869026:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829119.869029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869032:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829119.869034:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829119.869041:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829119.869043:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869045:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829119.869050:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829119.869054:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869057:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829119.869061:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829119.869063:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829119.869066:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829119.869068:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869069:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829119.869071:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829119.869073:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829119.869077:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829119.869081:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829119.869086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327312384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829119.869094:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829119.869096:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829119.869100:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327312384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829119.869120:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829119.869123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829119.869126:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829119.869129:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829119.869130:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829119.869133:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829119.869136:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3267
00000400:00000200:3.0:1787829119.869140:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829119.869144:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829119.869146:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829119.869149:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869152:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.869153:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869156:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829119.869158:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829119.869159:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829119.869161:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829119.869163:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829119.869166:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829119.869168:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829119.869171:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869173:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.869175:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869177:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829119.869179:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829119.869182:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869184:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829119.869186:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829119.869189:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829119.869192:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869193:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829119.869196:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829119.869199:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869201:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829119.869207:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829119.869209:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829119.869212:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829119.869213:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869214:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829119.869216:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829119.869218:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829119.869222:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829119.869225:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829119.869229:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327312512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829119.869237:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829119.869238:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829119.869242:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327312512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829119.869251:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829119.869253:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829119.869254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1787829119.869257:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00080000:3.0:1787829119.869258:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829119.869259:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829119.869261:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829119.869264:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829119.869266:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3267
00000400:00000200:3.0:1787829119.869269:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829119.869272:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829119.869274:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829119.869277:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869279:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.869281:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869284:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829119.869286:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829119.869288:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829119.869290:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829119.869292:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829119.869294:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829119.869297:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829119.869300:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869302:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829119.869305:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869306:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829119.869309:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829119.869312:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829119.869315:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829119.869317:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829119.869319:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829119.869321:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869324:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829119.869326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829119.869328:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869330:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829119.869334:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829119.869336:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829119.869338:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829119.869340:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829119.869342:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829119.869343:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829119.869345:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829119.869348:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829119.869351:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829119.869357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327312640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829119.869365:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829119.869367:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829119.869371:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327312640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829119.869380:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829119.869385:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3277)
00000800:00000200:1.1:1787829119.869651:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829119.869653:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787829119.869876:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829119.869878:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787829119.870058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829119.870076:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000001:2.0:1787829120.380760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829120.380765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829120.380767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829120.380770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829120.380773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829120.380776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829120.380778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829120.380780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829120.380782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829120.572751:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297940652, at 4297940672
00000100:00000040:3.0:1787829120.572753:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297940652, at 4297940672
00000100:00000010:3.0:1787829120.572767:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e636c0.
00000100:00000010:2.0:1787829120.572775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63c60.
00000100:00000001:2.0:1787829121.404765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829121.404771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829121.404772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829121.404776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829121.404779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829121.404781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829121.404783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829121.404785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829121.404787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829121.724832:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829121.724839:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829121.724843:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829122.428754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829122.428768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829122.428770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829122.428774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829122.428777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829122.428780:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829122.428783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829122.428786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829122.428788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829122.748945:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829122.748950:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829122.748954:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829124.796843:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829124.796847:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829124.796851:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1787829124.989419:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829124.989424:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787829124.989947:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829124.989949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787829124.990200:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829124.990202:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787829124.990384:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829124.990385:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000100:00000040:3.0:1787829125.692749:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297945772, at 4297945792
00000100:00000010:3.0:1787829125.692762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000100:00000040:2.0:1787829125.692791:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297945772, at 4297945792
00000100:00000010:2.0:1787829125.692807:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000020:00000001:2.0:1787829125.820858:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829125.820862:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829125.820865:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829127.868830:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829127.868836:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829127.868841:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829128.892878:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829128.892882:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829128.892886:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829130.108785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829130.108793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829130.108796:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829130.108799:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829130.108803:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829130.108808:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3277
00000400:00000200:2.0:1787829130.108816:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829130.108821:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829130.108823:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829130.108826:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108829:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.108830:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108848:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829130.108851:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829130.108853:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829130.108855:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829130.108857:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829130.108859:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829130.108861:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829130.108863:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108864:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.108865:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108866:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829130.108868:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829130.108871:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108872:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829130.108875:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829130.108877:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829130.108879:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.108881:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829130.108882:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829130.108889:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829130.108891:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829130.108894:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829130.108896:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.108898:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829130.108900:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829130.108903:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829130.108911:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829130.108916:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829130.108923:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327313280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829130.108930:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829130.108934:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829130.108937:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327313280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829130.108955:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829130.108957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829130.108959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829130.108960:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829130.108961:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829130.108963:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829130.108965:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3277
00000400:00000200:2.0:1787829130.108967:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829130.108969:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829130.108970:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829130.108972:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108974:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.108976:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108978:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829130.108979:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829130.108980:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829130.108981:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829130.108983:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829130.108984:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829130.108986:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829130.108988:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108989:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.108991:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.108993:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829130.108997:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829130.108999:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109002:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829130.109005:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829130.109011:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829130.109017:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109022:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829130.109030:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829130.109036:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109054:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829130.109062:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787829130.109068:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829130.109090:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829130.109093:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109095:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829130.109100:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829130.109104:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829130.109110:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829130.109116:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829130.109125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327313408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829130.109136:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829130.109141:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829130.109148:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327313408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829130.109175:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829130.109179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829130.109185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829130.109190:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829130.109191:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829130.109193:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829130.109199:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3277
00000400:00000200:2.0:1787829130.109205:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829130.109211:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829130.109217:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829130.109224:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109229:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.109235:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109242:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829130.109249:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829130.109253:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829130.109259:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829130.109265:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829130.109269:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829130.109274:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829130.109281:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109285:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.109290:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109294:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829130.109297:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829130.109304:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109307:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829130.109309:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829130.109314:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829130.109317:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109319:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829130.109324:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829130.109328:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109331:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829130.109334:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:2.0:1787829130.109336:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000800:00000200:0.1:1787829130.109338:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000020:00000040:2.0:1787829130.109341:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829130.109346:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829130.109350:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109352:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829130.109354:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829130.109356:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829130.109360:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829130.109378:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829130.109400:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327313536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829130.109421:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829130.109424:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829130.109432:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327313536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829130.109445:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829130.109449:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829130.109451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829130.109455:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829130.109457:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829130.109465:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829130.109469:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3277
00000400:00000200:2.0:1787829130.109475:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829130.109481:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829130.109484:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829130.109503:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109506:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.109512:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109517:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829130.109522:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829130.109525:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829130.109527:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829130.109532:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829130.109535:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829130.109540:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829130.109546:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109550:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829130.109556:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109561:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829130.109565:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829130.109571:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829130.109575:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829130.109578:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829130.109582:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829130.109587:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109590:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829130.109593:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829130.109598:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109601:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829130.109608:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787829130.109613:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829130.109620:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829130.109624:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829130.109628:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829130.109631:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829130.109634:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829130.109639:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829130.109646:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829130.109685:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327313664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829130.109700:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829130.109704:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829130.109713:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327313664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829130.109730:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829130.109737:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3287)
00000800:00000200:0.1:1787829130.109898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829130.109899:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787829130.110122:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829130.110124:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787829130.110333:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829130.110335:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000100:00000040:3.0:1787829130.812750:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297950892, at 4297950912
00000100:00000040:2.0:1787829130.812752:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297950892, at 4297950912
00000100:00000010:3.0:1787829130.812769:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000100:00000010:2.0:1787829130.812773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000020:00000001:2.0:1787829130.940885:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829130.940891:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829130.940895:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829131.964857:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829131.964861:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829131.964866:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829134.012856:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829134.012861:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829134.012865:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829135.036914:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829135.036920:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829135.036924:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829135.229240:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829135.229244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829135.229750:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829135.229752:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829135.229983:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829135.229985:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829135.230184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829135.230186:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:3.0:1787829135.932742:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297956012, at 4297956032
00000100:00000040:2.0:1787829135.932743:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297956012, at 4297956032
00000100:00000010:3.0:1787829135.932757:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63720.
00000100:00000010:2.0:1787829135.932760:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e634e0.
00000020:00000001:2.0:1787829137.084806:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829137.084809:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829137.084812:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829138.108882:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829138.108886:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829138.108890:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829140.156792:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829140.156796:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829140.156800:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829140.349332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829140.349337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829140.349969:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829140.349972:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829140.350165:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829140.350166:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829140.350339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829140.350341:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787829141.052775:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297961132, at 4297961152
00000100:00000040:3.0:1787829141.052775:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297961132, at 4297961152
00000100:00000010:3.0:1787829141.052793:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63360.
00000100:00000010:2.0:1787829141.052794:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e638a0.
00000020:00000001:2.0:1787829141.180864:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829141.180869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829141.180874:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829143.228810:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829143.228814:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829143.228817:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829144.252873:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829144.252877:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829144.252880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829145.468804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829145.468813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829145.468816:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829145.468819:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829145.468824:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829145.468830:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3293
00000400:00000200:3.0:1787829145.468855:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829145.468861:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829145.468864:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829145.468867:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.468869:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.468871:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.468873:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829145.468876:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829145.468878:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829145.468881:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829145.468884:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829145.468886:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829145.468888:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829145.468890:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.468891:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.468893:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.468894:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829145.468896:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829145.468899:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829145.468900:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829145.468903:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829145.468906:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829145.468908:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.468909:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829145.468911:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829145.468919:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829145.468922:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829145.468925:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829145.468927:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.468929:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829145.468931:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829145.468934:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829145.468941:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829145.468946:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829145.468953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327314816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829145.468962:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829145.468966:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829145.468969:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327314816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829145.468988:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829145.468992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829145.469011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829145.469013:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829145.469015:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829145.469019:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829145.469022:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3293
00000400:00000200:3.0:1787829145.469028:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829145.469033:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829145.469037:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829145.469042:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469045:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.469049:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469055:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829145.469063:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829145.469067:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829145.469069:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829145.469074:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829145.469080:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829145.469085:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829145.469092:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469098:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.469103:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469107:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829145.469111:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829145.469118:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469142:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829145.469145:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829145.469152:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829145.469156:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469159:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829145.469163:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829145.469169:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469181:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829145.469186:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829145.469190:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829145.469193:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829145.469194:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469201:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829145.469205:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829145.469208:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829145.469215:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829145.469222:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829145.469230:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327314944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829145.469243:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829145.469247:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829145.469271:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327314944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829145.469280:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829145.469285:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829145.469291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829145.469296:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829145.469299:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829145.469302:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829145.469309:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3293
00000400:00000200:3.0:1787829145.469317:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829145.469326:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829145.469329:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829145.469338:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469343:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.469348:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469352:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829145.469356:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829145.469359:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829145.469363:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829145.469366:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829145.469372:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829145.469377:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829145.469414:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469417:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.469419:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469421:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829145.469427:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829145.469430:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469432:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829145.469435:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829145.469439:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829145.469442:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469448:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829145.469451:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.1:1787829145.469452:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:3.0:1787829145.469471:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469475:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829145.469477:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000010:3.0:1787829145.469480:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829145.469487:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829145.469494:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829145.469499:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469502:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829145.469507:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829145.469521:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829145.469528:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829145.469536:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829145.469548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327315072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829145.469572:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829145.469576:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829145.469593:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327315072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829145.469605:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829145.469611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829145.469632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829145.469635:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829145.469637:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829145.469640:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829145.469642:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3293
00000400:00000200:3.0:1787829145.469647:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829145.469653:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829145.469711:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829145.469716:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469719:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.469722:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469726:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829145.469750:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829145.469754:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829145.469757:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829145.469761:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829145.469767:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829145.469774:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829145.469781:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469788:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829145.469790:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469794:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829145.469801:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829145.469804:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829145.469807:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829145.469813:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829145.469817:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829145.469819:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469820:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829145.469837:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829145.469839:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469840:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829145.469844:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829145.469850:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829145.469853:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829145.469855:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829145.469859:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829145.469861:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829145.469863:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829145.469871:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829145.469876:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829145.469884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327315200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829145.469892:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829145.469895:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829145.469903:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327315200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829145.469922:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829145.469927:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3303)
00000800:00000200:1.1:1787829145.470339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829145.470343:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829145.470898:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829145.470902:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829145.471330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829145.471348:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1787829146.172734:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297966253, at 4297966272
00000100:00000040:3.0:1787829146.172734:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297966253, at 4297966272
00000100:00000010:2.0:1787829146.172749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f60.
00000100:00000010:3.0:1787829146.172753:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ae0.
00000020:00000001:2.0:1787829146.300892:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829146.300896:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829146.300900:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829147.324824:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829147.324827:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829147.324830:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829149.372826:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829149.372829:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829149.372832:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829150.396928:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829150.396934:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829150.396938:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829150.588791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829150.588815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829150.588819:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829150.588823:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829150.588827:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829150.588835:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3298
00000400:00000200:3.0:1787829150.588844:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829150.588850:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829150.588854:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829150.588857:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.588860:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.588863:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.588865:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829150.588869:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829150.588870:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829150.588873:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829150.588877:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829150.588879:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829150.588881:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829150.588884:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.588886:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.588888:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.588890:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829150.588892:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829150.588895:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829150.588898:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829150.588900:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829150.588903:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829150.588906:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.588909:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829150.588911:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829150.588920:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829150.588923:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829150.588927:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829150.588930:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.588934:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829150.588937:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829150.588942:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829150.588952:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829150.588961:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829150.588974:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327315328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829150.589003:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829150.589008:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829150.589016:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327315328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829150.589042:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829150.589045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829150.589048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829150.589049:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829150.589051:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829150.589054:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829150.589056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3298
00000400:00000200:3.0:1787829150.589060:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829150.589065:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829150.589067:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829150.589070:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589073:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.589075:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589077:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829150.589080:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829150.589082:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829150.589084:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829150.589086:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829150.589090:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829150.589092:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829150.589095:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589098:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.589100:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589102:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829150.589104:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829150.589108:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589110:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829150.589115:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829150.589121:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829150.589126:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589130:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829150.589135:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829150.589147:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589155:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829150.589166:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829150.589176:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829150.589188:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829150.589192:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589199:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829150.589206:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829150.589213:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829150.589224:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829150.589231:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829150.589245:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327315456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829150.589259:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829150.589263:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829150.589268:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327315456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829150.589291:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829150.589298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829150.589308:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829150.589314:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829150.589317:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829150.589323:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829150.589332:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3298
00000400:00000200:3.0:1787829150.589344:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829150.589377:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829150.589382:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829150.589394:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589403:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.589414:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589422:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829150.589428:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829150.589432:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829150.589436:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829150.589441:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829150.589446:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829150.589453:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829150.589462:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589465:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.589472:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589479:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829150.589484:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829150.589494:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589499:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829150.589504:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829150.589508:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829150.589513:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589517:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829150.589520:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829150.589526:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589532:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829150.589539:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:3.0:1787829150.589540:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000800:00000200:1.1:1787829150.589545:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000020:00000040:3.0:1787829150.589547:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829150.589555:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829150.589559:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589563:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829150.589566:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829150.589570:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829150.589575:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829150.589584:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829150.589595:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327315584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829150.589610:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829150.589614:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829150.589625:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327315584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829150.589651:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829150.589683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829150.589687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829150.589689:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829150.589691:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829150.589696:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829150.589701:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3298
00000400:00000200:3.0:1787829150.589718:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829150.589733:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829150.589742:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829150.589751:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589760:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.589769:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589776:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829150.589785:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829150.589789:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829150.589793:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829150.589799:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829150.589806:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829150.589814:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829150.589825:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589833:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829150.589840:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589848:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829150.589855:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829150.589866:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829150.589877:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829150.589883:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829150.589891:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829150.589899:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589905:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829150.589911:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829150.589917:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589922:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829150.589929:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829150.589937:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829150.589947:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829150.589951:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829150.589955:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829150.589960:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829150.589964:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829150.589975:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829150.589985:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829150.590000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327315712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829150.590035:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829150.590043:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829150.590058:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327315712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829150.590091:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829150.590104:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3308)
00000800:00000200:1.1:1787829150.590313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829150.590315:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829150.590613:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829150.590617:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829150.591087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829150.591090:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000040:3.0:1787829151.292761:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297971373, at 4297971392
00000100:00000040:2.0:1787829151.292764:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297971373, at 4297971392
00000100:00000010:2.0:1787829151.292780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63060.
00000100:00000010:3.0:1787829151.292781:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63240.
00000100:00000001:2.0:1787829152.124747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829152.124753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829152.124756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829152.124762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829152.124767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829152.124770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829152.124773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829152.124776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829152.124780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829152.444880:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829152.444884:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829152.444886:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829153.148750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829153.148755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829153.148756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829153.148760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829153.148763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829153.148766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829153.148769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829153.148770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829153.148773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829153.468814:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829153.468818:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829153.468821:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829154.172747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829154.172753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829154.172755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829154.172759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829154.172761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829154.172764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829154.172766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829154.172768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829154.172770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829155.196738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829155.196744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829155.196747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829155.196752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829155.196756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829155.196759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829155.196762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829155.196765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829155.196769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829155.516795:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829155.516798:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829155.516801:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829155.708757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829155.708784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829155.708788:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829155.708794:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829155.708801:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829155.708810:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3303
00000400:00000200:3.0:1787829155.708823:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829155.708833:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829155.708839:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829155.708860:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.708866:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.708870:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.708875:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829155.708882:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829155.708885:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829155.708890:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829155.708895:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829155.708900:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829155.708905:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829155.708912:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.708917:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.708921:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.708925:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829155.708947:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829155.708950:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829155.708953:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829155.708956:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829155.708959:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829155.708962:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.708965:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829155.708967:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829155.708977:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829155.708981:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829155.708984:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829155.708987:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.708990:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829155.708993:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829155.708997:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829155.709005:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829155.709012:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829155.709021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327315840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829155.709043:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829155.709048:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829155.709054:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327315840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829155.709079:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829155.709082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829155.709086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829155.709088:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829155.709090:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829155.709093:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829155.709096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3303
00000400:00000200:3.0:1787829155.709101:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829155.709104:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829155.709107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829155.709111:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709113:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.709117:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709119:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829155.709125:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829155.709128:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829155.709133:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829155.709137:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829155.709143:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829155.709150:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829155.709159:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709165:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.709171:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709178:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829155.709184:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829155.709193:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709199:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829155.709204:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829155.709227:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829155.709238:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.709244:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829155.709255:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829155.709268:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.709274:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829155.709284:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829155.709298:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829155.709310:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829155.709355:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.709359:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829155.709363:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829155.709367:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829155.709383:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829155.709390:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829155.709417:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327315968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829155.709452:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829155.709462:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829155.709480:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327315968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829155.709525:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829155.709538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829155.709551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:1.1:1787829155.709552:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829155.709560:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829155.709566:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829155.709571:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829155.709577:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3303
00000400:00000200:3.0:1787829155.709587:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829155.709599:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829155.709628:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829155.709635:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709639:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.709644:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709649:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829155.709655:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829155.709661:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829155.709667:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829155.709672:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829155.709678:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829155.709682:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829155.709687:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709690:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.709717:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709724:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829155.709730:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829155.709739:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829155.709746:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829155.709753:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829155.709759:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829155.709765:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.709772:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829155.709776:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829155.709784:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.709790:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829155.709797:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829155.709803:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829155.709810:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829155.709814:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.709820:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829155.709824:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829155.709827:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829155.709853:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829155.709864:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829155.709884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327316096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829155.709918:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829155.709943:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829155.709952:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327316096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829155.709971:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829155.709976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829155.709985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829155.709990:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829155.709993:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829155.709999:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829155.710006:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3303
00000400:00000200:3.0:1787829155.710015:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:1.1:1787829155.710015:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:3.0:1787829155.710025:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829155.710029:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829155.710032:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.710036:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.710039:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.710042:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829155.710047:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829155.710050:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829155.710054:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829155.710057:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829155.710063:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829155.710067:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829155.710074:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.710078:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829155.710084:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829155.710089:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829155.710096:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829155.710100:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829155.710105:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829155.710109:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829155.710115:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829155.710120:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.710125:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829155.710130:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829155.710135:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.710139:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829155.710145:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829155.710150:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829155.710157:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829155.710161:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829155.710166:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829155.710169:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829155.710174:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829155.710180:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829155.710188:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829155.710197:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327316224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829155.710212:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829155.710217:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829155.710225:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327316224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829155.710240:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829155.710249:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3313)
00000800:00000200:1.1:1787829155.710342:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829155.710502:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829156.220716:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829156.220725:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829156.220727:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829156.220732:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829156.220736:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829156.220740:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829156.220743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829156.220746:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829156.220749:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829156.412712:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297976492, at 4297976512
00000100:00000040:3.0:1787829156.412713:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297976492, at 4297976512
00000100:00000010:2.0:1787829156.412727:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63840.
00000100:00000010:3.0:1787829156.412731:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e631e0.
00000020:00000001:2.0:1787829156.540828:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829156.540831:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829156.540834:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829157.244787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829157.244793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829157.244795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829157.244800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829157.244803:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829157.244806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829157.244808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829157.244811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829157.244813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829158.268742:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829158.268747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829158.268749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829158.268754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829158.268757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829158.268759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829158.268761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829158.268763:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829158.268766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829158.588853:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829158.588857:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829158.588860:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829158.780901:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829158.780937:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829158.780944:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829158.780948:0:2336:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829158.780951:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829158.780956:0:2336:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829158.780959:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829158.780962:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:2.0:1787829158.780966:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829158.780970:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829158.780975:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98550
00000400:00000010:2.0:1787829158.780977:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98550.
00000100:00000001:2.0:1787829158.780981:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829158.780987:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0b40 x1874671327315840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781010:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829158.781015:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327315840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781024:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829158.781026:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829158.781037:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829158.781040:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955200.
00000400:00000200:2.0:1787829158.781044:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829158.781048:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829158.781052:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f983b8
00000400:00000010:2.0:1787829158.781053:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f983b8.
00000100:00000001:2.0:1787829158.781057:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829158.781064:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed21c0 x1874671327315968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781078:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829158.781094:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327315968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781108:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829158.781112:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829158.781122:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829158.781126:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:2.0:1787829158.781132:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829158.781141:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829158.781150:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98d48
00000400:00000010:2.0:1787829158.781154:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98d48.
00000100:00000001:2.0:1787829158.781159:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829158.781169:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0780 x1874671327316096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781187:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829158.781197:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327316096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781219:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829158.781224:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829158.781235:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829158.781241:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955900.
00000400:00000200:2.0:1787829158.781249:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829158.781256:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829158.781262:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98ee0
00000400:00000010:2.0:1787829158.781266:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98ee0.
00000100:00000001:2.0:1787829158.781270:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829158.781279:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed30c0 x1874671327316224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781313:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829158.781321:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327316224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829158.781339:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829158.781343:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829159.292751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829159.292760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829159.292762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829159.292768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829159.292772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829159.292776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829159.292779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829159.292782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829159.292786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829159.612854:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829159.612858:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829159.612863:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829160.316778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829160.316786:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829160.316789:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829160.316795:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829160.316799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829160.316803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829160.316806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829160.316809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829160.316812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829160.829485:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829160.829494:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787829160.830179:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829160.830181:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787829160.830448:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829160.830450:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787829160.830744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829160.830746:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787829161.340798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829161.340804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829161.340806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829161.340812:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829161.340815:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829161.340818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829161.340820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829161.340823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829161.340826:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829161.532734:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297981613, at 4297981632
00000100:00000040:2.0:1787829161.532742:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297981613, at 4297981632
00000100:00000010:3.0:1787829161.532749:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63de0.
00000100:00000010:2.0:1787829161.532753:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63420.
00000020:00000001:2.0:1787829161.660916:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829161.660924:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829161.660931:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829162.364854:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829162.364864:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829162.364868:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829162.364876:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829162.364882:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829162.364887:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829162.364891:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829162.364913:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829162.364919:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829162.684812:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829162.684816:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829162.684818:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829163.388769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829163.388774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829163.388776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829163.388789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829163.388792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829163.388795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829163.388797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829163.388799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829163.388801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829164.412760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829164.412766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829164.412767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829164.412772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829164.412775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829164.412779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829164.412782:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829164.412786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829164.412790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829164.732823:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829164.732826:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829164.732830:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829165.436788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829165.436793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829165.436795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829165.436799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829165.436802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829165.436805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829165.436807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829165.436809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829165.436811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829165.756925:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829165.756930:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829165.756934:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829165.949572:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829165.949598:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829165.950207:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829165.950210:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829165.950664:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829165.950668:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829165.951067:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829165.951070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829166.460796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829166.460801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829166.460803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829166.460807:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829166.460811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829166.460814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829166.460816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829166.460818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829166.460821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829166.652759:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297986733, at 4297986752
00000100:00000010:3.0:1787829166.652771:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e635a0.
00000100:00000040:2.0:1787829166.652858:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297986733, at 4297986752
00000100:00000010:2.0:1787829166.652862:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ea0.
00000100:00000001:2.0:1787829167.484737:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829167.484743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829167.484744:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829167.484747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829167.484750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829167.484753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829167.484755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829167.484756:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829167.484759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829167.804987:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829167.804991:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829167.804995:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829168.508762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829168.508768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829168.508771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829168.508777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829168.508781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829168.508784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829168.508788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829168.508790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829168.508793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829168.828865:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829168.828869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829168.828871:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829169.532784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829169.532791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829169.532793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829169.532799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829169.532803:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829169.532807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829169.532810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829169.532813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829169.532816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829170.556791:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829170.556800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829170.556802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829170.556808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829170.556812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829170.556816:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829170.556819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829170.556822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829170.556824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829170.876964:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829170.876970:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829170.876974:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787829171.069308:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829171.069314:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829171.069766:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829171.069768:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829171.069975:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829171.069977:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829171.070244:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829171.070247:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1787829171.070354:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327317760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829196 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829171.070362:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829171.070364:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070366:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829171.070367:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070369:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829171.070371:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829171.070374:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829171.070377:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829171.070380:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829171.070384:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327317760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829196 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829171.070393:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327317760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829171.070397:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070399:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829171.070402:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327317504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829171.070409:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829171.070411:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829171.070413:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829171.070416:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829171.070418:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829171.070419:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829171.070421:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829171.070423:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829171.070426:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829171.070427:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829171.070429:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829171.070432:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829171.070433:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070435:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829171.070439:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327317632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829171.070446:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829171.070448:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829171.070449:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829171.070452:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829171.070453:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829171.070455:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829171.070457:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829171.070458:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829171.070460:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829171.070462:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829171.070463:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829171.070465:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829171.070466:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070468:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829171.070471:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327317760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829171.070478:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829171.070479:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829171.070480:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829171.070482:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829171.070482:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829171.070483:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829171.070485:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829171.070486:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829171.070487:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829171.070488:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829171.070489:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829171.070490:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829171.070491:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070492:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829171.070494:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829171.070495:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829171.070496:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829171.070498:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829171.580744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829171.580752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829171.580755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829171.580762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829171.580767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829171.580772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829171.580775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829171.580778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829171.580782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829171.772733:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297991852, at 4297991872
00000100:00000010:3.0:1787829171.772750:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a20.
00000100:00000040:2.0:1787829171.772752:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297991852, at 4297991872
00000100:00000010:2.0:1787829171.772765:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f00.
00000020:00000001:2.0:1787829171.900883:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829171.900888:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829171.900893:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829172.604807:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829172.604815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829172.604817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829172.604824:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829172.604828:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829172.604833:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829172.604836:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829172.604839:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829172.604842:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829173.628744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829173.628752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829173.628754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829173.628760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829173.628764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829173.628768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829173.628770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829173.628773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829173.628777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829173.948829:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829173.948835:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829173.948839:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829174.652769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829174.652777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829174.652779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829174.652783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829174.652786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829174.652790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829174.652792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829174.652794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829174.652796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829174.972893:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829174.972899:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829174.972904:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829175.676848:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829175.676859:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829175.676862:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829175.676870:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829175.676875:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829175.676879:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829175.676883:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829175.676886:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829175.676890:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.189054:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829176.189062:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829176.189067:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829176.189080:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189107:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829176.189110:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.189115:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829176.189119:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829176.189126:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829176.189134:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327317888:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829176.189144:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829176.189151:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829176.189159:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.189168:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829176.189194:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829176.189218:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829176.189234:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829176.189246:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829176.189260:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829176.189275:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327317888, portal 25
00000100:00000001:1.0:1787829176.189282:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829176.189286:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829176.189301:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189319:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829176.189325:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829176.189332:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829176.189338:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327317888, offset 0
00000400:00000200:1.0:1787829176.189348:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829176.189364:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.189377:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829176.189391:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829176.189403:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829176.189416:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829176.189427:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829176.189440:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40255:40255:256:4294967295] 192.168.203.116@tcp LPNI seq info [40255:40255:8:4294967295]
00000400:00000200:1.0:1787829176.189447:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.189466:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829176.189483:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.189494:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829176.189510:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829176.189519:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829176.189533:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829176.189540:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829176.189553:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.189557:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829176.189567:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829176.189579:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829176.189588:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829176.189592:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829176.189600:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829176.189610:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189632:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.189641:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189659:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.189661:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829176.189667:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829176.189731:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829176.189735:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4297996973
00000100:00000010:1.0:1787829176.189741:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d80.
00000100:00000001:1.0:1787829176.189747:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829176.189752:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4297996973
00000100:00000010:1.0:1787829176.189756:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63b40.
00000100:00000001:1.0:1787829176.189759:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829176.189763:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829176.189766:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.189768:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829176.189774:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829176/real 1787829176]  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829176.189783:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829176.189785:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829176.189786:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829176.189789:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829176.189791:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829176.189793:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829176.189796:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829176.189803:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189809:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829176.189810:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.189812:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.189813:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829176.189817:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189822:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829176.189826:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189832:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829176.189835:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.189836:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.189837:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.189840:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829176.189841:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829176.189844:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829176.189848:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829176.189850:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829176.189853:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829176.189860:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327317888:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829176.189864:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.189866:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.189868:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327317888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189873:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829176.189875:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829176.189877:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829176.189880:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829176.189881:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829176.189883:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829176.189885:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829176.189887:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829176.189890:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829176.189891:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829176.189893:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829176.189896:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829176.189897:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.189899:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.189904:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829176.189906:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.189907:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829176.189909:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829176.189910:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829176.189913:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189924:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829176.189925:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.189927:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829176.189929:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829176.189931:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829176.189933:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318016:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829176.189935:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829176.189937:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829176.189938:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.189940:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829176.189942:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829176.189954:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829176.189957:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829176.189959:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829176.189961:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829176.189964:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318016, portal 10
00000100:00000001:1.0:1787829176.189966:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829176.189967:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829176.189972:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.189978:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829176.189980:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829176.189982:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829176.189984:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327318016, offset 0
00000400:00000200:1.0:1787829176.189987:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829176.189990:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.189993:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829176.189996:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829176.189998:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829176.190000:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829176.190002:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829176.190005:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40256:40256:256:4294967295] 192.168.203.116@tcp LPNI seq info [40256:40256:8:4294967295]
00000400:00000200:1.0:1787829176.190009:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.190039:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829176.190044:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.190047:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829176.190050:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829176.190053:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829176.190057:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829176.190059:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829176.190063:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.190065:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829176.190067:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829176.190071:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829176.190073:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829176.190075:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829176.190077:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829176.190081:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190105:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.190107:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190113:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190114:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829176.190117:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829176.190120:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829176.190123:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829176.190125:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297996299
00000100:00000001:1.0:1787829176.190127:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297996973
00000100:00000001:1.0:1787829176.190129:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297996299
00000100:00000001:1.0:1787829176.190131:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297996973
00000100:00000001:1.0:1787829176.190132:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829176.190134:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190136:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829176.190139:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829176/real 1787829176]  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829176.190145:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829176.190147:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829176.190148:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829176.190149:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829176.190151:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829176.190152:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829176.190156:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829176.190192:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190217:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829176.190219:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190233:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.190235:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829176.190241:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190251:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829176.190257:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190267:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829176.190270:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190272:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.190273:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190275:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829176.190277:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829176.190279:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829176.190282:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829176.190284:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829176.190288:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829176.190294:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318016:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829176.190298:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829176.190300:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190306:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829176.190307:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.190309:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829176.190310:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829176.190312:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829176.190314:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318144:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829176.190317:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829176.190319:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829176.190320:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.190322:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829176.190325:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829176.190326:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829176.190330:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829176.190332:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829176.190335:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829176.190338:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318144, portal 4
00000100:00000001:1.0:1787829176.190339:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829176.190340:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829176.190346:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190351:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829176.190354:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829176.190355:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829176.190358:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327318144, offset 0
00000400:00000200:1.0:1787829176.190361:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829176.190365:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.190368:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829176.190371:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829176.190374:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829176.190378:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829176.190381:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829176.190385:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40257:40257:256:4294967295] 192.168.203.116@tcp LPNI seq info [40257:40257:8:4294967295]
00000400:00000200:1.0:1787829176.190391:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.190400:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829176.190407:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.190411:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829176.190416:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829176.190420:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829176.190426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829176.190430:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829176.190435:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.190438:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829176.190442:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829176.190446:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829176.190449:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829176.190450:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829176.190453:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829176.190457:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190463:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.190466:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190472:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190473:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829176.190477:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829176.190480:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829176.190483:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829176.190484:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297996306
00000100:00000001:1.0:1787829176.190487:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297996973
00000100:00000001:1.0:1787829176.190489:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297996306
00000100:00000001:1.0:1787829176.190490:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297996973
00000100:00000001:1.0:1787829176.190492:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829176.190494:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190495:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829176.190498:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829176/real 1787829176]  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829176.190505:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829176.190506:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829176.190507:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829176.190509:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829176.190511:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829176.190512:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829176.190515:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829176.190522:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190527:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829176.190528:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190530:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.190531:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829176.190534:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190539:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829176.190542:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190548:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829176.190550:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190552:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.190553:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190554:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829176.190556:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829176.190558:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829176.190560:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829176.190562:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829176.190565:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829176.190571:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318144:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829176.190574:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829176.190576:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190581:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829176.190583:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.190584:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829176.190586:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829176.190587:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829176.190590:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829176.190609:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829176.190610:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829176.190612:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829176.190613:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829176.190616:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829176.190618:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829176.190620:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829176.190623:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829176.190625:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829176.190627:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318272, portal 4
00000100:00000001:1.0:1787829176.190629:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829176.190630:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829176.190635:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190640:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829176.190642:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829176.190644:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829176.190646:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327318272, offset 0
00000400:00000200:1.0:1787829176.190649:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829176.190652:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.190655:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829176.190657:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829176.190659:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829176.190661:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829176.190663:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829176.190666:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40258:40258:256:4294967295] 192.168.203.116@tcp LPNI seq info [40258:40258:8:4294967295]
00000400:00000200:1.0:1787829176.190669:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829176.190674:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829176.190678:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.190680:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829176.190683:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829176.190703:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829176.190707:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829176.190710:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829176.190713:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829176.190714:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829176.190717:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829176.190720:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829176.190722:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829176.190724:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829176.190726:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829176.190729:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190735:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.190738:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190743:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190744:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829176.190746:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829176.190748:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829176.190751:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829176.190753:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4297996312
00000100:00000001:1.0:1787829176.190755:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4297996973
00000100:00000001:1.0:1787829176.190757:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4297996312
00000100:00000001:1.0:1787829176.190759:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4297996973
00000100:00000001:1.0:1787829176.190761:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829176.190762:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190764:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829176.190766:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829176/real 1787829176]  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829176.190772:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829176.190773:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829176.190774:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829176.190776:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829176.190777:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829176.190779:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829176.190782:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829176.190788:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190798:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829176.190799:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190801:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.190802:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829176.190805:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190809:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829176.190812:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190817:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829176.190819:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190820:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829176.190821:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190823:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829176.190824:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829176.190826:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829176.190828:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829176.190830:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829176.190832:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829176.190838:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829176.190841:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190842:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.190845:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327318016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190850:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829176.190852:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829176.190853:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829176.190855:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829176.190856:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829176.190858:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829176.190860:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829176.190861:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829176.190863:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829176.190865:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829176.190866:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829176.190868:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829176.190869:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190871:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.190873:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327318144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190878:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829176.190879:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829176.190880:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829176.190882:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829176.190883:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829176.190884:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829176.190885:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829176.190886:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829176.190888:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829176.190888:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829176.190889:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829176.190891:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829176.190892:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190893:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829176.190896:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327318272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829176.190901:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829176.190902:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829176.190903:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829176.190904:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829176.190905:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829176.190906:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829176.190908:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829176.190909:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829176.190911:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829176.190912:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829176.190913:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829176.190915:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829176.190915:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190917:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829176.190920:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829176.190938:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829176.190940:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829176.190941:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829176.700750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829176.700756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829176.700758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829176.700762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829176.700765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829176.700768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829176.700770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829176.700772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829176.700774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829176.892716:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4297996973, at 4297996992
00000100:00000010:3.0:1787829176.892730:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63b40.
00000100:00000040:2.0:1787829176.892737:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4297996973, at 4297996992
00000100:00000010:2.0:1787829176.892747:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d80.
00000020:00000001:2.0:1787829177.020889:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829177.020893:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829177.020896:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829177.724765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829177.724770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829177.724772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829177.724775:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829177.724778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829177.724781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829177.724783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829177.724785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829177.724787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829178.044902:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829178.044906:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829178.044910:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829178.748746:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829178.748754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829178.748756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829178.748761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829178.748764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829178.748768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829178.748770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829178.748773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829178.748775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829179.772723:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829179.772731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829179.772734:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829179.772739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829179.772742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829179.772746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829179.772748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829179.772751:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829179.772753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829180.092858:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829180.092862:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829180.092866:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829180.796770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829180.796779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829180.796782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829180.796788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829180.796794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829180.796798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829180.796800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829180.796803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829180.796806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829181.116861:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829181.116865:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829181.116868:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829181.308775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829181.308790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829181.308792:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829181.308797:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829181.308802:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829181.308810:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3328
00000400:00000200:2.0:1787829181.308820:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829181.308826:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829181.308830:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829181.308833:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.308835:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829181.308837:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.308839:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829181.308842:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829181.308844:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829181.308848:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829181.308852:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829181.308854:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829181.308857:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829181.308860:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.308863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829181.308865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.308868:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829181.308872:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829181.308876:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829181.308880:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829181.308883:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829181.308888:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829181.308890:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.308892:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829181.308895:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829181.308908:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829181.308913:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829181.308918:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829181.308920:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.308924:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829181.308927:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829181.308933:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829181.308944:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829181.308952:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829181.308964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829181.308974:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829181.308978:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829181.308982:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829181.309002:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829181.309005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829181.309008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829181.309009:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829181.309011:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829181.309013:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829181.309014:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3328
00000400:00000200:2.0:1787829181.309018:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829181.309020:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829181.309022:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829181.309025:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309027:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829181.309029:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309031:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829181.309033:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829181.309034:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829181.309036:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829181.309038:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829181.309039:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829181.309041:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787829181.309041:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829181.309047:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829181.309048:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829181.309049:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829181.309052:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829181.309053:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309057:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829181.309059:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:1.0:1787829181.309063:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829181.309064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309068:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829181.309073:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829181.309079:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787829181.309080:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829181.309082:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829181.309083:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309086:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829181.309088:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1787829181.309091:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829181.309092:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829181.309098:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787829181.309098:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309103:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829181.309105:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327318400:192.168.203.116@tcp:250:kworker.0
00000100:00000010:2.0:1787829181.309107:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000100:00000001:1.0:1787829181.309111:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787829181.309112:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829181.309116:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829181.309117:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829181.309120:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.309121:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309122:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829181.309124:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787829181.309125:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829181.309129:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829181.309132:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000010:2.0:1787829181.309146:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
02000000:00000001:1.0:1787829181.309147:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309154:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.309158:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:2.0:1787829181.309163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787829181.309164:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000100:00000001:2.0:1787829181.309172:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787829181.309173:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000001:2.0:1787829181.309175:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829181.309180:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829181.309183:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318400, portal 25
00000100:00000001:2.0:1787829181.309188:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829181.309191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829181.309194:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1787829181.309195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829181.309197:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00000001:2.0:1787829181.309198:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829181.309201:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829181.309204:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1787829181.309208:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829181.309209:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3328
00000400:00000200:2.0:1787829181.309217:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829181.309223:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829181.309227:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787829181.309227:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829181.309234:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829181.309236:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309236:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829181.309240:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829181.309242:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000001:2.0:1787829181.309243:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829181.309248:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829181.309250:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829181.309251:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787829181.309253:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327318400, offset 0
00000020:00000040:2.0:1787829181.309253:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829181.309256:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829181.309261:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829181.309263:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309264:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829181.309268:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829181.309270:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309271:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829181.309276:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829181.309278:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309280:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829181.309281:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829181.309283:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829181.309285:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.309287:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829181.309287:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829181.309290:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829181.309291:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309292:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829181.309295:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787829181.309299:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829181.309301:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829181.309302:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309303:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829181.309304:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787829181.309308:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:2.0:1787829181.309308:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829181.309313:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829181.309316:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.309319:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:2.0:1787829181.309321:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829181.309328:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787829181.309329:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829181.309333:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.309338:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1787829181.309339:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829181.309348:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309349:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829181.309353:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829181.309358:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40259:40259:256:4294967295] 192.168.203.116@tcp LPNI seq info [40259:40259:8:4294967295]
00000100:00080000:2.0:1787829181.309359:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829181.309361:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829181.309363:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829181.309367:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829181.309368:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829181.309371:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3328
00000400:00000200:1.0:1787829181.309382:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787829181.309395:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:1.0:1787829181.309397:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829181.309401:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829181.309403:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000010:1.0:1787829181.309404:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00000040:2.0:1787829181.309406:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309410:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829181.309411:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309413:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:1.0:1787829181.309416:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787829181.309417:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829181.309421:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000200:1.0:1787829181.309423:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1787829181.309424:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829181.309427:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829181.309430:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:1.1:1787829181.309432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:2.0:1787829181.309433:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829181.309440:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829181.309440:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1787829181.309447:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:1.0:1787829181.309449:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.309454:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000040:2.0:1787829181.309454:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829181.309459:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829181.309460:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1787829181.309464:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:1.0:1787829181.309469:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1787829181.309474:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829181.309475:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829181.309480:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:2.0:1787829181.309480:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829181.309486:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829181.309487:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787829181.309490:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1787829181.309494:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829181.309496:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309502:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829181.309506:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829181.309509:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000008:00000001:2.0:1787829181.309509:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309511:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829181.309516:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787829181.309517:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829181.309522:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829181.309527:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829181.309529:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.309532:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829181.309533:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829181.309534:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309537:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829181.309539:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:1.0:1787829181.309541:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829181.309547:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309548:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.309556:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:2.0:1787829181.309558:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829181.309559:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298002092
00000100:00000010:1.0:1787829181.309567:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63480.
00000100:00000001:2.0:1787829181.309575:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829181.309576:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787829181.309579:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309583:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298002092
00000100:00000010:1.0:1787829181.309589:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d20.
00000100:00000040:2.0:1787829181.309589:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309594:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829181.309599:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829181.309603:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309604:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309609:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787829181.309609:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3339)
00000100:00100000:1.0:1787829181.309615:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829181/real 1787829181]  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829181.309624:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829181.309626:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829181.309629:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829181.309633:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829181.309635:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829181.309639:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829181.309643:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829181.309653:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309685:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829181.309687:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309689:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.309691:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829181.309695:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309701:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829181.309704:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309710:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829181.309712:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309713:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.309714:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309716:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829181.309717:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829181.309719:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829181.309722:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829181.309723:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829181.309727:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829181.309733:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327318400:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829181.309737:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309739:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.309741:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327318400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309745:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829181.309747:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829181.309750:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829181.309752:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829181.309753:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829181.309755:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829181.309757:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829181.309759:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:1.0:1787829181.309762:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829181.309765:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829181.309766:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:1.0:1787829181.309769:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829181.309770:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309772:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309779:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829181.309781:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309782:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829181.309783:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829181.309785:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829181.309787:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309792:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829181.309793:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.309795:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829181.309797:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829181.309798:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829181.309800:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318528:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829181.309802:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829181.309803:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829181.309804:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.309805:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829181.309807:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829181.309809:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.309812:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829181.309814:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829181.309817:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829181.309819:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318528, portal 10
00000100:00000001:1.0:1787829181.309820:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829181.309822:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:1.0:1787829181.309827:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309832:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829181.309834:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829181.309836:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829181.309837:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327318528, offset 0
00000400:00000200:1.0:1787829181.309840:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829181.309843:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829181.309846:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829181.309848:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829181.309850:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829181.309852:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829181.309854:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829181.309856:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40260:40260:256:4294967295] 192.168.203.116@tcp LPNI seq info [40260:40260:8:4294967295]
00000400:00000200:1.0:1787829181.309859:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829181.309864:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829181.309867:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.309869:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829181.309872:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829181.309874:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829181.309878:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829181.309880:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829181.309896:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.309898:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829181.309900:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829181.309903:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829181.309906:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829181.309907:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829181.309909:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829181.309912:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309917:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.309919:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309924:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309925:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829181.309928:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829181.309931:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829181.309933:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829181.309935:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298001419
00000100:00000001:1.0:1787829181.309937:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298002092
00000100:00000001:1.0:1787829181.309939:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298001419
00000100:00000001:1.0:1787829181.309941:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298002092
00000100:00000001:1.0:1787829181.309942:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829181.309944:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.309945:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829181.309947:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829181/real 1787829181]  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829181.309952:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829181.309953:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829181.309954:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829181.309956:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829181.309956:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829181.309958:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829181.309960:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829181.309965:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309969:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829181.309970:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309972:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.309973:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829181.309975:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309979:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829181.309981:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.309986:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829181.309988:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309989:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.309990:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.309991:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829181.309992:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829181.309994:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829181.309996:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829181.309997:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829181.310000:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829181.310004:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318528:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829181.310006:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829181.310008:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310012:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829181.310013:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.310015:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829181.310016:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829181.310017:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829181.310019:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318656:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829181.310021:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829181.310022:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829181.310023:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.310024:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829181.310027:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829181.310028:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.310031:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829181.310033:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829181.310035:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829181.310037:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318656, portal 4
00000100:00000001:1.0:1787829181.310038:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829181.310039:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:1.0:1787829181.310043:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310047:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829181.310048:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829181.310050:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829181.310052:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327318656, offset 0
00000400:00000200:1.0:1787829181.310054:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829181.310056:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829181.310059:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829181.310061:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829181.310063:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829181.310065:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829181.310067:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829181.310071:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40261:40261:256:4294967295] 192.168.203.116@tcp LPNI seq info [40261:40261:8:4294967295]
00000400:00000200:1.0:1787829181.310075:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829181.310092:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829181.310097:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.310101:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829181.310106:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829181.310108:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829181.310114:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829181.310116:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829181.310120:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.310122:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829181.310126:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829181.310129:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829181.310145:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829181.310157:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829181.310162:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829181.310166:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310175:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.310179:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310185:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.310186:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829181.310189:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829181.310191:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829181.310194:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829181.310196:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298001426
00000100:00000001:1.0:1787829181.310197:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298002092
00000100:00000001:1.0:1787829181.310199:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298001426
00000100:00000001:1.0:1787829181.310201:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298002092
00000100:00000001:1.0:1787829181.310202:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829181.310203:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.310205:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829181.310207:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829181/real 1787829181]  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829181.310213:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829181.310214:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829181.310215:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829181.310217:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829181.310218:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829181.310219:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829181.310222:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829181.310227:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310239:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829181.310240:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310242:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.310244:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829181.310248:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310256:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829181.310260:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310267:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829181.310270:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310272:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.310273:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310275:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829181.310277:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829181.310280:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829181.310283:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829181.310286:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829181.310290:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829181.310300:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318656:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829181.310304:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829181.310307:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310311:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829181.310312:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.310314:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829181.310316:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829181.310317:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829181.310319:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829181.310322:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829181.310323:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829181.310324:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829181.310326:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829181.310329:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829181.310330:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829181.310332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829181.310334:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829181.310337:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829181.310339:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318784, portal 4
00000100:00000001:1.0:1787829181.310341:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829181.310341:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:1.0:1787829181.310346:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310351:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829181.310352:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829181.310354:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829181.310356:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327318784, offset 0
00000400:00000200:1.0:1787829181.310358:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829181.310361:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829181.310363:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829181.310366:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829181.310367:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829181.310369:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829181.310371:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829181.310373:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40262:40262:256:4294967295] 192.168.203.116@tcp LPNI seq info [40262:40262:8:4294967295]
00000400:00000200:1.0:1787829181.310376:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829181.310381:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829181.310385:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.310387:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829181.310390:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829181.310392:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829181.310395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829181.310397:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829181.310399:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829181.310401:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829181.310403:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829181.310405:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829181.310407:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829181.310408:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829181.310410:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829181.310413:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310418:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.310420:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310424:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.310426:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829181.310428:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829181.310430:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829181.310432:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829181.310433:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298001431
00000100:00000001:1.0:1787829181.310435:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298002092
00000100:00000001:1.0:1787829181.310437:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298001431
00000100:00000001:1.0:1787829181.310438:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298002092
00000100:00000001:1.0:1787829181.310440:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829181.310441:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.310442:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829181.310444:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829181/real 1787829181]  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829181.310450:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829181.310451:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829181.310452:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829181.310454:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829181.310455:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829181.310456:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829181.310458:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829181.310463:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310467:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829181.310468:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310469:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.310470:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829181.310473:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310477:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829181.310479:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310483:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829181.310484:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310485:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829181.310486:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310487:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829181.310488:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829181.310490:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829181.310491:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829181.310493:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829181.310496:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829181.310501:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327318784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829181.310503:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310505:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.310506:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327318528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310510:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829181.310511:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829181.310512:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829181.310514:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829181.310515:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829181.310516:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829181.310517:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829181.310518:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:1.0:1787829181.310519:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829181.310520:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829181.310521:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:1.0:1787829181.310523:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829181.310524:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310525:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.310527:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327318656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310531:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829181.310532:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829181.310533:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829181.310534:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829181.310535:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829181.310536:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829181.310537:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829181.310538:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:1.0:1787829181.310539:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829181.310541:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829181.310542:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:1.0:1787829181.310544:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829181.310545:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310546:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829181.310548:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327318784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829181.310551:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829181.310552:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829181.310553:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829181.310554:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829181.310555:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829181.310556:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829181.310557:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829181.310558:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:1.0:1787829181.310559:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829181.310559:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829181.310560:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:1.0:1787829181.310562:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829181.310563:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310564:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829181.310566:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829181.310567:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829181.310568:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829181.310569:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829181.820793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829181.820801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829181.820803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829181.820809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829181.820813:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829181.820816:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829181.820818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829181.820822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829181.820825:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829182.012743:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298002092, at 4298002112
00000100:00000040:2.0:1787829182.012754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298002092, at 4298002112
00000100:00000010:3.0:1787829182.012763:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d20.
00000100:00000010:2.0:1787829182.012768:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63480.
00000100:00000001:2.0:1787829182.844752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829182.844760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829182.844762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829182.844767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829182.844771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829182.844774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829182.844777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829182.844780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829182.844782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829183.164865:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829183.164869:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829183.164873:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829183.868772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829183.868779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829183.868781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829183.868786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829183.868790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829183.868794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829183.868797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829183.868799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829183.868802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829184.188819:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829184.188823:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829184.188826:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829184.892771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829184.892780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829184.892782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829184.892786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829184.892790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829184.892793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829184.892795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829184.892797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829184.892800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829185.916775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829185.916781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829185.916783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829185.916787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829185.916789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829185.916793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829185.916795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829185.916797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829185.916800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829186.236876:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829186.236882:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829186.236886:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829186.429112:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829186.429120:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829186.429125:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829186.429156:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429184:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829186.429191:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.429203:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829186.429209:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829186.429217:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829186.429226:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327318912:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829186.429239:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829186.429245:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829186.429251:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.429257:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829186.429283:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829186.429289:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829186.429301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829186.429307:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829186.429316:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829186.429325:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327318912, portal 25
00000100:00000001:1.0:1787829186.429330:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829186.429332:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829186.429338:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429348:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829186.429353:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829186.429357:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829186.429361:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327318912, offset 0
00000400:00000200:1.0:1787829186.429374:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829186.429385:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.429395:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829186.429404:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829186.429412:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829186.429418:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829186.429423:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829186.429431:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40263:40263:256:4294967295] 192.168.203.116@tcp LPNI seq info [40263:40263:8:4294967295]
00000400:00000200:1.0:1787829186.429437:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.429452:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829186.429464:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.429470:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829186.429481:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829186.429505:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829186.429519:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829186.429527:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787829186.429539:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.429546:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829186.429557:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829186.429566:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829186.429574:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829186.429579:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829186.429586:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829186.429594:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429611:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.429617:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429636:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.429643:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829186.429654:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829186.429666:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829186.429671:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298007212
00000100:00000010:1.0:1787829186.429676:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63000.
00000100:00000001:1.0:1787829186.429683:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829186.429691:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298007213
00000100:00000010:1.0:1787829186.429694:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e639c0.
00000100:00000001:1.0:1787829186.429697:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829186.429700:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829186.429732:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.429737:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829186.429745:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829186]  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829186.429775:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829186.429780:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829186.429785:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829186.429794:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829186.429799:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829186.429804:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829186.429812:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829186.429826:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429839:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829186.429843:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.429846:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.429849:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829186.429856:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429868:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829186.429884:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.429899:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829186.429904:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.429910:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.429912:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.429915:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829186.429918:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829186.429923:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829186.429929:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829186.429931:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829186.429940:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829186.429961:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327318912:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829186.429978:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.429984:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.429991:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327318912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430005:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829186.430010:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829186.430014:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829186.430019:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829186.430024:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829186.430025:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829186.430028:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829186.430032:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829186.430036:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829186.430040:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829186.430041:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829186.430046:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829186.430048:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430053:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430064:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829186.430068:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430083:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829186.430086:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829186.430089:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829186.430095:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430125:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829186.430128:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.430131:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829186.430150:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829186.430155:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829186.430159:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319040:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829186.430162:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829186.430164:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829186.430165:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.430168:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829186.430171:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829186.430173:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829186.430177:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829186.430179:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829186.430182:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829186.430185:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319040, portal 10
00000100:00000001:1.0:1787829186.430186:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829186.430188:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829186.430194:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430217:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829186.430219:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829186.430220:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829186.430223:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327319040, offset 0
00000400:00000200:1.0:1787829186.430226:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829186.430229:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.430233:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829186.430235:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829186.430238:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829186.430240:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829186.430242:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829186.430245:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40264:40264:256:4294967295] 192.168.203.116@tcp LPNI seq info [40264:40264:8:4294967295]
00000400:00000200:1.0:1787829186.430248:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.430254:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829186.430257:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.430260:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829186.430264:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829186.430283:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829186.430287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829186.430289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787829186.430292:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.430294:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829186.430296:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829186.430299:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829186.430301:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829186.430303:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829186.430305:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829186.430307:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430313:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.430315:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430320:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430321:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829186.430324:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829186.430327:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829186.430329:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829186.430347:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298006539
00000100:00000001:1.0:1787829186.430349:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298007212
00000100:00000001:1.0:1787829186.430351:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298006539
00000100:00000001:1.0:1787829186.430353:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298007213
00000100:00000001:1.0:1787829186.430355:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829186.430357:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430358:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829186.430361:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829186]  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829186.430367:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829186.430368:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829186.430370:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829186.430372:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829186.430373:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829186.430375:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829186.430378:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829186.430384:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430389:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829186.430390:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430392:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.430393:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829186.430395:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430416:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829186.430419:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430423:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829186.430425:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430426:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.430427:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430428:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829186.430429:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829186.430431:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829186.430433:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829186.430435:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829186.430437:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829186.430443:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319040:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829186.430446:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829186.430448:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430457:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829186.430458:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.430460:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829186.430461:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829186.430463:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829186.430464:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829186.430466:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829186.430467:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829186.430469:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.430470:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829186.430472:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829186.430473:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829186.430492:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829186.430494:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829186.430498:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829186.430501:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319168, portal 4
00000100:00000001:1.0:1787829186.430502:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829186.430504:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829186.430511:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430517:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829186.430520:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829186.430522:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829186.430525:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327319168, offset 0
00000400:00000200:1.0:1787829186.430530:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829186.430534:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.430538:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829186.430541:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829186.430543:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829186.430546:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829186.430548:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829186.430552:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40265:40265:256:4294967295] 192.168.203.116@tcp LPNI seq info [40265:40265:8:4294967295]
00000400:00000200:1.0:1787829186.430556:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.430563:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829186.430568:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.430572:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829186.430575:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829186.430577:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829186.430580:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829186.430582:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787829186.430585:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.430587:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829186.430589:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829186.430591:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829186.430594:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829186.430595:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829186.430597:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829186.430599:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430605:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.430607:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430612:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430613:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829186.430615:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829186.430618:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829186.430622:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829186.430624:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298006546
00000100:00000001:1.0:1787829186.430627:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298007212
00000100:00000001:1.0:1787829186.430629:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298006546
00000100:00000001:1.0:1787829186.430631:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298007213
00000100:00000001:1.0:1787829186.430634:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829186.430637:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430639:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829186.430643:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829186]  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829186.430653:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829186.430655:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829186.430657:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829186.430660:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829186.430662:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829186.430664:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829186.430668:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829186.430675:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430680:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829186.430682:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430683:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.430684:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829186.430686:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430691:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829186.430694:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430699:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829186.430700:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430701:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.430702:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430703:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829186.430704:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829186.430706:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829186.430708:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829186.430710:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829186.430713:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829186.430718:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829186.430720:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430721:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.430723:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327319040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430727:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829186.430728:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829186.430729:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829186.430731:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829186.430732:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829186.430733:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829186.430735:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829186.430736:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:1.0:1787829186.430738:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829186.430739:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829186.430740:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829186.430742:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829186.430743:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430744:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.430746:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327319168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430774:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829186.430776:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829186.430778:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829186.430781:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829186.430782:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829186.430784:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829186.430786:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829186.430789:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829186.430791:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829186.430793:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829186.430795:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829186.430798:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829186.430799:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430801:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.430804:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829186.430806:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430807:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829186.430808:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829186.430810:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829186.430813:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430818:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829186.430819:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.430821:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829186.430822:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829186.430824:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829186.430826:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319296:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829186.430828:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829186.430829:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829186.430831:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829186.430832:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829186.430835:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829186.430836:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829186.430838:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829186.430840:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829186.430843:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829186.430845:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319296, portal 4
00000100:00000001:1.0:1787829186.430847:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829186.430848:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829186.430852:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430861:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829186.430863:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829186.430864:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829186.430866:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327319296, offset 0
00000400:00000200:1.0:1787829186.430868:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829186.430871:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.430873:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829186.430885:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829186.430886:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829186.430888:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829186.430890:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829186.430892:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40266:40266:256:4294967295] 192.168.203.116@tcp LPNI seq info [40266:40266:8:4294967295]
00000400:00000200:1.0:1787829186.430895:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829186.430900:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829186.430903:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.430905:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829186.430908:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829186.430910:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787829186.430912:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829186.430915:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787829186.430918:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829186.430919:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829186.430921:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829186.430924:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829186.430926:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829186.430927:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829186.430929:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829186.430931:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430936:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.430938:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.430951:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430963:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829186.430971:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829186.430974:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829186.430978:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829186.430980:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298006552
00000100:00000001:1.0:1787829186.430984:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298007212
00000100:00000001:1.0:1787829186.430986:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298006552
00000100:00000001:1.0:1787829186.430989:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298007213
00000100:00000001:1.0:1787829186.430991:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829186.430993:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.430996:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829186.430999:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829186]  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829186.431005:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829186.431006:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829186.431008:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829186.431010:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829186.431011:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829186.431012:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829186.431015:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829186.431020:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.431024:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829186.431025:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.431026:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.431027:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829186.431029:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.431035:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829186.431037:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.431042:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829186.431043:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.431044:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829186.431045:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.431046:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829186.431047:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829186.431049:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829186.431056:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829186.431058:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829186.431060:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829186.431065:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319296:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829186.431067:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.431068:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829186.431070:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327319296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829186.431074:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829186.431076:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829186.431077:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829186.431079:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829186.431080:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829186.431080:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829186.431082:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829186.431083:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:1.0:1787829186.431085:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829186.431086:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829186.431087:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829186.431089:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829186.431090:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.431091:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829186.431094:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829186.431095:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829186.431096:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829186.431109:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829186.940738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829186.940744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829186.940746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829186.940750:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829186.940753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829186.940756:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829186.940758:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829186.940760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829186.940762:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829187.132737:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298007213, at 4298007232
00000100:00000010:3.0:1787829187.132765:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e639c0.
00000100:00000040:2.0:1787829187.132782:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298007212, at 4298007232
00000100:00000010:2.0:1787829187.132790:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63000.
00000020:00000001:2.0:1787829187.260906:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829187.260911:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829187.260916:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829187.964766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829187.964773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829187.964775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829187.964780:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829187.964784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829187.964787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829187.964790:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829187.964793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829187.964796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829188.988747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829188.988755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829188.988758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829188.988764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829188.988768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829188.988772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829188.988774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829188.988777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829188.988780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829189.308868:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829189.308873:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829189.308878:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829190.012776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829190.012782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829190.012784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829190.012789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829190.012793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829190.012797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829190.012799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829190.012802:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829190.012805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829190.332918:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829190.332925:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829190.332931:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829191.036751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829191.036759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829191.036762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829191.036767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829191.036772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829191.036776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829191.036779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829191.036781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829191.036784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829191.548818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829191.548830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829191.548834:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829191.548838:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829191.548843:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829191.548863:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3339
00000400:00000200:3.0:1787829191.548875:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829191.548885:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829191.548890:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829191.548895:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.548900:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829191.548904:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.548907:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829191.548912:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829191.548915:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829191.548918:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829191.548922:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829191.548926:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829191.548929:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829191.548933:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.548935:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829191.548938:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.548941:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829191.548946:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829191.548950:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829191.548954:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829191.548957:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829191.548962:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829191.548966:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.548969:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829191.548972:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829191.548983:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829191.548988:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829191.548993:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829191.548995:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.548998:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829191.549001:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829191.549005:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829191.549014:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829191.549022:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829191.549033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829191.549048:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829191.549053:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829191.549059:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829191.549136:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829191.549141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829191.549145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829191.549146:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829191.549149:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829191.549153:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829191.549156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3339
00000400:00000200:3.0:1787829191.549161:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829191.549167:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829191.549171:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829191.549175:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.549179:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829191.549182:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.549185:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829191.549187:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1787829191.549189:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787829191.549193:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1787829191.549194:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:1.0:1787829191.549198:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787829191.549199:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829191.549204:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000040:1.0:1787829191.549209:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829191.549210:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829191.549218:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829191.549230:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829191.549232:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829191.549236:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.549238:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787829191.549240:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829191.549245:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787829191.549245:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.549252:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:1.0:1787829191.549253:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787829191.549260:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00100000:1.0:1787829191.549261:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327319424:192.168.203.116@tcp:250:kworker.0
00000100:00080000:3.0:1787829191.549271:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829191.549273:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829191.549279:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.549281:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787829191.549283:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829191.549287:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829191.549292:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829191.549293:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787829191.549300:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829191.549303:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:3.0:1787829191.549304:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829191.549310:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787829191.549311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829191.549328:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.549331:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.549335:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:3.0:1787829191.549339:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000400:00000010:1.0:1787829191.549343:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000020:00000040:3.0:1787829191.549345:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:1.0:1787829191.549352:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
02000000:00000001:3.0:1787829191.549352:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829191.549356:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.549361:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1787829191.549363:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319424, portal 25
02000000:00000001:3.0:1787829191.549365:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787829191.549367:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829191.549370:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
02000000:00000001:3.0:1787829191.549370:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829191.549383:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787829191.549383:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829191.549392:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.549395:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829191.549405:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00100000:3.0:1787829191.549406:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787829191.549408:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829191.549415:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327319424, offset 0
00000400:00000200:1.0:1787829191.549428:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829191.549429:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829191.549435:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.549439:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829191.549445:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829191.549448:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829191.549457:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829191.549465:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:3.0:1787829191.549472:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.549473:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829191.549479:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:3.0:1787829191.549481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829191.549487:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40267:40267:256:4294967295] 192.168.203.116@tcp LPNI seq info [40267:40267:8:4294967295]
00000100:00080000:3.0:1787829191.549492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1787829191.549496:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829191.549497:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829191.549502:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829191.549510:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829191.549514:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1787829191.549517:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3339
00000800:00000200:1.0:1787829191.549530:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787829191.549530:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829191.549538:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:1.0:1787829191.549541:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00080000:3.0:1787829191.549543:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829191.549550:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.549556:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787829191.549557:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829191.549562:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829191.549566:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:3.0:1787829191.549568:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829191.549576:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829191.549578:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829191.549583:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.1:1787829191.549584:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787829191.549588:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:1.1:1787829191.549590:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000020:00000001:3.0:1787829191.549593:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000100:1.0:1787829191.549600:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829191.549601:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:1.0:1787829191.549604:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000040:3.0:1787829191.549611:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829191.549613:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787829191.549617:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829191.549623:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1787829191.549626:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829191.549639:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829191.549640:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000020:00000040:3.0:1787829191.549644:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000010:1.0:1787829191.549645:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00080000:3.0:1787829191.549652:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829191.549653:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787829191.549711:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829191.549714:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829191.549729:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829191.549733:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.549735:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829191.549740:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829191.549745:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829191.549747:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829191.549749:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829191.549758:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000100:00000001:1.0:1787829191.549764:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:3.0:1787829191.549764:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829191.549771:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1787829191.549772:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829191.549776:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.549782:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787829191.549798:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829191.549798:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829191.549801:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.549803:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:3.0:1787829191.549809:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:1.0:1787829191.549814:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.549815:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.549828:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:3.0:1787829191.549830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829191.549836:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298012333
00000100:00000001:3.0:1787829191.549847:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:1.0:1787829191.549849:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63000.
00000100:00000001:3.0:1787829191.549851:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829191.549867:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.549871:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829191.549881:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298012333
00000100:00000001:3.0:1787829191.549883:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829191.549889:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e639c0.
00000100:00080000:3.0:1787829191.549890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829191.549897:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787829191.549898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829191.549901:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829191.549903:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787829191.549907:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829191.549908:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829191.549911:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1787829191.549916:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:1.0:1787829191.549921:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191]  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829191.549924:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3339
00000400:00000200:3.0:1787829191.549937:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000001:1.0:1787829191.549943:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:3.0:1787829191.549944:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000001:1.0:1787829191.549947:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:3.0:1787829191.549949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1787829191.549951:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000100:00000040:3.0:1787829191.549957:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829191.549958:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829191.549963:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:3.0:1787829191.549965:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829191.549968:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1787829191.549970:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:1.0:1787829191.549976:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829191.549977:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829191.549984:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829191.549988:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829191.549992:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829191.549997:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000200:1.0:1787829191.549998:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829191.550004:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829191.550009:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:1.0:1787829191.550016:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:3.0:1787829191.550018:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829191.550019:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550025:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787829191.550025:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829191.550030:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829191.550032:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00100000:1.0:1787829191.550038:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829191.550039:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829191.550047:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829191.550056:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829191.550057:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829191.550079:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829191.550083:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829191.550085:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829191.550091:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829191.550097:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550108:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787829191.550109:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829191.550114:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:3.0:1787829191.550116:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829191.550118:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.550120:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550126:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000008:00000001:3.0:1787829191.550126:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550130:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787829191.550132:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829191.550135:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000010:3.0:1787829191.550139:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000100:00080000:1.0:1787829191.550140:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829191.550146:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000040:3.0:1787829191.550147:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000040:1.0:1787829191.550152:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829191.550154:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829191.550158:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829191.550163:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829191.550168:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:1.0:1787829191.550182:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327319424:192.168.203.116@tcp:250:kworker.0
02000000:00000001:3.0:1787829191.550182:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829191.550191:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:1.0:1787829191.550192:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550198:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787829191.550201:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829191.550203:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327319424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787829191.550212:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829191.550213:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829191.550217:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829191.550221:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829191.550227:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1787829191.550230:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000020:00000001:1.0:1787829191.550231:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829191.550235:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000100:00000001:3.0:1787829191.550237:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829191.550239:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829191.550244:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
00000100:00000040:3.0:1787829191.550248:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829191.550249:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829191.550253:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829191.550256:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:1.0:1787829191.550263:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829191.550268:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829191.550270:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550276:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829191.550280:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3349)
00000100:00000001:1.0:1787829191.550288:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829191.550293:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550297:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829191.550300:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829191.550303:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829191.550310:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550330:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829191.550333:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.550336:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829191.550339:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829191.550341:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829191.550344:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319552:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829191.550347:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829191.550349:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829191.550351:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.550352:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829191.550355:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829191.550374:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.550377:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829191.550380:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829191.550383:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829191.550386:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319552, portal 10
00000100:00000001:1.0:1787829191.550388:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829191.550389:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:1.0:1787829191.550394:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550399:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829191.550401:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829191.550403:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829191.550405:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327319552, offset 0
00000400:00000200:1.0:1787829191.550408:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829191.550411:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829191.550415:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829191.550418:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829191.550420:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829191.550422:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829191.550424:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829191.550442:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40268:40268:256:4294967295] 192.168.203.116@tcp LPNI seq info [40268:40268:8:4294967295]
00000400:00000200:1.0:1787829191.550446:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829191.550452:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829191.550456:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829191.550459:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829191.550463:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829191.550465:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829191.550469:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829191.550471:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829191.550475:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829191.550476:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829191.550479:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829191.550482:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829191.550485:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829191.550486:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829191.550488:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829191.550491:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550498:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829191.550501:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550506:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550508:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829191.550510:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829191.550514:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829191.550517:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829191.550518:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298011659
00000100:00000001:1.0:1787829191.550521:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298012333
00000100:00000001:1.0:1787829191.550524:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298011659
00000100:00000001:1.0:1787829191.550527:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298012333
00000100:00000001:1.0:1787829191.550530:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829191.550533:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550535:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829191.550540:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191]  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829191.550550:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829191.550552:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829191.550554:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829191.550558:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829191.550560:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829191.550563:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829191.550567:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829191.550579:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550588:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829191.550590:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550591:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.550593:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829191.550596:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550602:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829191.550605:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550611:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829191.550613:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550615:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.550616:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550618:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829191.550620:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829191.550622:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829191.550625:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829191.550627:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829191.550630:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829191.550636:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319552:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829191.550640:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829191.550642:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550647:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829191.550649:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.550651:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829191.550653:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829191.550654:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829191.550657:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319680:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829191.550674:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829191.550677:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829191.550678:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.550681:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829191.550685:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829191.550688:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.550692:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829191.550695:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829191.550722:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829191.550727:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319680, portal 4
00000100:00000001:1.0:1787829191.550729:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829191.550731:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:1.0:1787829191.550738:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550754:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829191.550756:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829191.550758:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829191.550761:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327319680, offset 0
00000400:00000200:1.0:1787829191.550764:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829191.550768:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829191.550771:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829191.550773:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829191.550775:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829191.550777:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829191.550779:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829191.550781:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40269:40269:256:4294967295] 192.168.203.116@tcp LPNI seq info [40269:40269:8:4294967295]
00000400:00000200:1.0:1787829191.550785:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829191.550789:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829191.550793:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829191.550795:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829191.550798:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829191.550800:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829191.550804:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829191.550807:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829191.550822:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829191.550825:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829191.550829:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829191.550833:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829191.550837:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829191.550839:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829191.550842:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829191.550846:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550856:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829191.550860:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550869:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550871:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829191.550875:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829191.550885:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829191.550888:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829191.550890:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298011667
00000100:00000001:1.0:1787829191.550892:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298012333
00000100:00000001:1.0:1787829191.550894:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298011667
00000100:00000001:1.0:1787829191.550896:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298012333
00000100:00000001:1.0:1787829191.550897:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829191.550899:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.550900:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829191.550903:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191]  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829191.550909:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829191.550910:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829191.550912:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829191.550914:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829191.550915:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829191.550916:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829191.550919:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829191.550925:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550930:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829191.550931:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550933:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.550934:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829191.550937:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550942:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829191.550945:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550950:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829191.550951:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550952:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.550953:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.550955:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829191.550956:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829191.550957:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829191.550960:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829191.550961:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829191.550964:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829191.550969:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319680:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829191.550972:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829191.550974:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.550979:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829191.550980:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.550982:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829191.550984:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829191.550985:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829191.550987:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319808:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829191.550989:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829191.550991:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829191.550993:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829191.550994:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829191.550997:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829191.550998:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829191.551001:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829191.551003:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829191.551005:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829191.551007:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319808, portal 4
00000100:00000001:1.0:1787829191.551008:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829191.551009:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:1.0:1787829191.551014:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551018:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829191.551020:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829191.551021:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829191.551023:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327319808, offset 0
00000400:00000200:1.0:1787829191.551026:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829191.551029:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829191.551031:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829191.551034:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829191.551036:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829191.551038:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829191.551039:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829191.551042:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40270:40270:256:4294967295] 192.168.203.116@tcp LPNI seq info [40270:40270:8:4294967295]
00000400:00000200:1.0:1787829191.551045:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829191.551050:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829191.551053:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829191.551055:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829191.551058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829191.551060:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829191.551080:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829191.551082:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829191.551085:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829191.551087:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829191.551090:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829191.551092:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829191.551095:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829191.551096:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829191.551098:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829191.551101:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551107:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829191.551109:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551114:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.551115:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829191.551118:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829191.551120:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829191.551123:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829191.551124:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298011672
00000100:00000001:1.0:1787829191.551126:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298012333
00000100:00000001:1.0:1787829191.551128:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298011672
00000100:00000001:1.0:1787829191.551129:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298012333
00000100:00000001:1.0:1787829191.551131:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829191.551132:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.551134:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829191.551136:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191]  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829191.551142:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829191.551143:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829191.551144:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829191.551146:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829191.551147:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829191.551149:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829191.551151:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829191.551157:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551161:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829191.551162:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551164:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.551165:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829191.551167:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551171:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829191.551174:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551179:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829191.551181:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551182:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829191.551183:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551185:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829191.551186:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829191.551188:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829191.551190:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829191.551191:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829191.551194:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829191.551199:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327319808:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829191.551201:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551203:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829191.551205:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327319552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551210:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829191.551212:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829191.551213:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829191.551215:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829191.551216:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829191.551217:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829191.551218:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829191.551220:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:1.0:1787829191.551222:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829191.551223:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829191.551224:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:1.0:1787829191.551226:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829191.551227:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551229:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829191.551231:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327319680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551236:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829191.551237:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829191.551238:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829191.551239:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829191.551240:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829191.551241:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829191.551242:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829191.551243:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:1.0:1787829191.551244:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829191.551245:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829191.551246:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:1.0:1787829191.551247:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829191.551248:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551249:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829191.551251:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327319808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829191.551255:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829191.551257:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829191.551258:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829191.551259:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829191.551260:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829191.551261:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829191.551263:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829191.551264:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:1.0:1787829191.551265:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829191.551266:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829191.551268:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:1.0:1787829191.551270:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829191.551271:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551273:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829191.551278:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829191.551279:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829191.551281:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829191.551283:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829192.060780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829192.060789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829192.060792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829192.060798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829192.060803:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829192.060808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829192.060812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829192.060815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829192.060820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829192.252783:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298012333, at 4298012352
00000100:00000040:3.0:1787829192.252783:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298012333, at 4298012352
00000100:00000010:3.0:1787829192.252797:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e639c0.
00000100:00000010:2.0:1787829192.252827:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63000.
00000020:00000001:2.0:1787829192.381051:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829192.381073:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829192.381078:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829193.084739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829193.084747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829193.084749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829193.084754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829193.084758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829193.084762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829193.084765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829193.084768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829193.084770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829193.404927:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829193.404933:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829193.404939:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829194.108774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829194.108782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829194.108783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829194.108788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829194.108792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829194.108795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829194.108798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829194.108800:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829194.108803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829195.132789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829195.132798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829195.132801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829195.132807:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829195.132813:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829195.132817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829195.132820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829195.132823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829195.132827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829195.452925:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829195.452933:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829195.452940:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829196.156774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829196.156781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829196.156783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829196.156787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829196.156792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829196.156806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829196.156809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829196.156812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829196.156815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829196.476958:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829196.476969:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829196.476977:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829196.668988:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829196.668994:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829196.668999:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829196.669010:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669044:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829196.669047:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.669051:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829196.669057:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829196.669061:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829196.669068:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327319936:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829196.669076:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829196.669082:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829196.669086:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.669092:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829196.669099:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829196.669104:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829196.669113:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829196.669118:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829196.669128:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829196.669136:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327319936, portal 25
00000100:00000001:1.0:1787829196.669141:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829196.669145:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829196.669157:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669178:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829196.669186:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829196.669191:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829196.669198:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327319936, offset 0
00000400:00000200:1.0:1787829196.669209:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829196.669221:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.669241:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829196.669250:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829196.669257:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829196.669265:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829196.669270:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829196.669278:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40271:40271:256:4294967295] 192.168.203.116@tcp LPNI seq info [40271:40271:8:4294967295]
00000400:00000200:1.0:1787829196.669288:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.669312:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829196.669327:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.669334:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829196.669344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829196.669352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829196.669361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829196.669366:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829196.669375:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.669378:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829196.669385:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829196.669390:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829196.669397:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829196.669402:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829196.669407:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829196.669415:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669434:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.669443:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669458:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.669462:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829196.669470:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829196.669480:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829196.669485:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298017452
00000100:00000010:1.0:1787829196.669490:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63480.
00000100:00000001:1.0:1787829196.669500:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829196.669509:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298017452
00000100:00000010:1.0:1787829196.669516:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d20.
00000100:00000001:1.0:1787829196.669522:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829196.669534:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829196.669543:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.669550:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829196.669563:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196]  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829196.669592:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829196.669596:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829196.669620:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829196.669629:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829196.669635:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829196.669640:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829196.669650:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829196.669726:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669738:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829196.669742:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.669745:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.669747:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829196.669751:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669757:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829196.669761:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669773:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829196.669775:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.669777:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.669778:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.669780:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829196.669782:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829196.669784:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829196.669788:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829196.669790:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829196.669793:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829196.669808:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327319936:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829196.669812:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.669814:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.669816:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327319936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669822:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829196.669824:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829196.669826:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829196.669828:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829196.669830:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829196.669832:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829196.669834:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829196.669836:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829196.669839:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829196.669841:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829196.669842:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829196.669845:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829196.669846:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.669848:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.669853:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829196.669855:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.669857:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829196.669858:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829196.669859:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829196.669862:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669868:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829196.669869:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.669871:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829196.669873:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829196.669874:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829196.669877:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320064:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829196.669879:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829196.669881:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829196.669882:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.669884:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829196.669886:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829196.669888:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829196.669892:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829196.669894:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829196.669897:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829196.669899:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320064, portal 10
00000100:00000001:1.0:1787829196.669901:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829196.669902:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829196.669907:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.669913:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829196.669915:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829196.669917:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829196.669919:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327320064, offset 0
00000400:00000200:1.0:1787829196.669922:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829196.669925:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.669928:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829196.669931:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829196.669933:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829196.669935:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829196.669937:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829196.669939:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40272:40272:256:4294967295] 192.168.203.116@tcp LPNI seq info [40272:40272:8:4294967295]
00000400:00000200:1.0:1787829196.669943:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.669948:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829196.669952:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.669955:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829196.669958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829196.669960:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829196.669964:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829196.669966:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829196.669981:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.669983:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829196.669985:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829196.669988:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829196.670007:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829196.670009:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829196.670012:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829196.670016:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670024:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.670047:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670092:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.670094:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829196.670099:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829196.670104:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829196.670108:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829196.670111:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298016779
00000100:00000001:1.0:1787829196.670126:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298017452
00000100:00000001:1.0:1787829196.670129:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298016779
00000100:00000001:1.0:1787829196.670132:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298017452
00000100:00000001:1.0:1787829196.670134:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829196.670137:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.670139:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829196.670144:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196]  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829196.670153:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829196.670155:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829196.670157:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829196.670160:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829196.670163:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829196.670165:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829196.670169:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829196.670179:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670193:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829196.670195:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.670197:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.670198:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829196.670202:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670211:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829196.670216:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670224:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829196.670226:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.670228:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.670230:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.670232:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829196.670234:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829196.670237:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829196.670240:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829196.670243:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829196.670247:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829196.670256:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320064:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829196.670260:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829196.670263:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670271:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829196.670273:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.670275:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829196.670278:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829196.670281:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829196.670284:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320192:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829196.670287:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829196.670296:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829196.670298:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.670301:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829196.670305:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829196.670307:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829196.670311:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829196.670314:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829196.670318:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829196.670321:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320192, portal 4
00000100:00000001:1.0:1787829196.670323:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829196.670342:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829196.670348:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670366:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829196.670368:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829196.670370:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829196.670373:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327320192, offset 0
00000400:00000200:1.0:1787829196.670377:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829196.670381:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.670385:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829196.670388:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829196.670391:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829196.670393:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829196.670396:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829196.670399:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40273:40273:256:4294967295] 192.168.203.116@tcp LPNI seq info [40273:40273:8:4294967295]
00000400:00000200:1.0:1787829196.670403:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.670410:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829196.670414:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.670417:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829196.670421:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829196.670424:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829196.670428:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829196.670431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829196.670435:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.670437:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829196.670440:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829196.670444:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829196.670447:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829196.670449:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829196.670451:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829196.670455:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670463:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.670466:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670473:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.670474:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829196.670477:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829196.670480:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829196.670483:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829196.670485:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298016786
00000100:00000001:1.0:1787829196.670488:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298017452
00000100:00000001:1.0:1787829196.670490:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298016786
00000100:00000001:1.0:1787829196.670492:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298017452
00000100:00000001:1.0:1787829196.670494:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829196.670496:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.670498:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829196.670502:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196]  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829196.670509:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829196.670510:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829196.670512:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829196.670514:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829196.670516:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829196.670518:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829196.670521:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829196.670529:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670535:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829196.670537:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.670538:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.670540:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829196.670543:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670552:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829196.670558:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670585:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829196.670590:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.670592:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.670594:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.670597:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829196.670600:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829196.670605:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829196.670611:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829196.670615:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829196.670623:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829196.670648:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320192:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829196.670656:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829196.670662:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670673:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829196.670675:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.670678:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829196.670681:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829196.670683:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829196.670687:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320320:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829196.670708:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829196.670710:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829196.670712:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829196.670715:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829196.670742:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829196.670745:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829196.670749:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829196.670751:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829196.670755:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829196.670758:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320320, portal 4
00000100:00000001:1.0:1787829196.670760:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829196.670762:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829196.670785:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670793:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829196.670796:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829196.670798:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829196.670801:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327320320, offset 0
00000400:00000200:1.0:1787829196.670806:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829196.670811:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.670814:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829196.670827:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829196.670830:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829196.670833:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829196.670836:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829196.670840:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40274:40274:256:4294967295] 192.168.203.116@tcp LPNI seq info [40274:40274:8:4294967295]
00000400:00000200:1.0:1787829196.670845:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829196.670853:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829196.670859:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.670862:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829196.670867:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829196.670871:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829196.670875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829196.670879:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829196.670883:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829196.670885:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829196.670889:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829196.670893:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829196.670897:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829196.670899:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829196.670901:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829196.670905:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670915:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.670918:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.670926:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.670928:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829196.670932:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829196.670935:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829196.670938:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829196.670941:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298016792
00000100:00000001:1.0:1787829196.670961:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298017452
00000100:00000001:1.0:1787829196.670963:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298016792
00000100:00000001:1.0:1787829196.670965:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298017452
00000100:00000001:1.0:1787829196.670967:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829196.670970:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.670972:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829196.670976:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196]  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829196.670984:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829196.670985:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829196.670986:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829196.670989:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829196.670990:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829196.670992:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829196.670995:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829196.671004:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.671010:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829196.671011:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671013:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.671014:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829196.671017:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.671023:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829196.671045:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.671052:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829196.671066:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671068:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829196.671069:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671071:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829196.671073:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829196.671075:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829196.671079:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829196.671082:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829196.671086:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829196.671094:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320320:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829196.671097:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671099:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.671103:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327320064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.671109:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829196.671111:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829196.671113:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829196.671115:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829196.671117:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829196.671119:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829196.671130:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829196.671132:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829196.671135:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829196.671136:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829196.671138:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829196.671140:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829196.671142:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671144:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.671147:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327320192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.671153:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829196.671154:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829196.671156:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829196.671158:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829196.671160:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829196.671162:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829196.671182:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829196.671186:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:1.0:1787829196.671189:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829196.671192:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829196.671194:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829196.671199:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829196.671201:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671204:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829196.671211:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327320320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829196.671224:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829196.671227:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829196.671230:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829196.671234:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829196.671237:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829196.671239:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829196.671242:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829196.671245:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829196.671249:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829196.671251:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829196.671254:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829196.671258:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829196.671260:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671264:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829196.671270:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829196.671273:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829196.671277:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829196.671280:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829197.180750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829197.180758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829197.180761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829197.180768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829197.180773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829197.180777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829197.180781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829197.180785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829197.180789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829197.372752:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298017452, at 4298017472
00000100:00000040:3.0:1787829197.372752:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298017452, at 4298017472
00000100:00000010:3.0:1787829197.372767:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d20.
00000100:00000010:2.0:1787829197.372771:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63480.
00000100:00000001:2.0:1787829198.204736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829198.204744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829198.204747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829198.204752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829198.204762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829198.204765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829198.204768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829198.204771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829198.204774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829198.524960:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829198.524967:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829198.524972:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829199.228755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829199.228761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829199.228762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829199.228767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829199.228770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829199.228773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829199.228775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829199.228778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829199.228780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829199.548887:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829199.548892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829199.548896:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829200.252785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829200.252793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829200.252795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829200.252801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829200.252805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829200.252810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829200.252813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829200.252816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829200.252819:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829201.276759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829201.276764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829201.276766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829201.276770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829201.276773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829201.276776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829201.276778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829201.276780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829201.276782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829201.596855:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829201.596859:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829201.596863:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829201.788962:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829201.788967:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829201.788972:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829201.788982:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789010:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829201.789012:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.789018:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829201.789021:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829201.789025:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829201.789031:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327320448:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829201.789037:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829201.789041:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829201.789045:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.789050:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829201.789056:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829201.789061:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829201.789070:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829201.789074:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829201.789083:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829201.789091:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320448, portal 25
00000100:00000001:1.0:1787829201.789097:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829201.789100:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829201.789110:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789128:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829201.789133:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829201.789137:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829201.789141:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327320448, offset 0
00000400:00000200:1.0:1787829201.789150:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829201.789161:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.789170:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829201.789177:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829201.789183:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829201.789188:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829201.789194:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829201.789201:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40275:40275:256:4294967295] 192.168.203.116@tcp LPNI seq info [40275:40275:8:4294967295]
00000400:00000200:1.0:1787829201.789207:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.789222:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829201.789231:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.789239:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829201.789251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829201.789260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829201.789275:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829201.789284:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829201.789298:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.789304:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829201.789312:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829201.789320:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829201.789327:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829201.789332:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829201.789339:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829201.789346:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789367:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.789375:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789391:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789393:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829201.789400:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829201.789409:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829201.789413:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298022572
00000100:00000010:1.0:1787829201.789421:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d80.
00000100:00000001:1.0:1787829201.789430:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829201.789435:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298022572
00000100:00000010:1.0:1787829201.789442:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63b40.
00000100:00000001:1.0:1787829201.789448:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829201.789460:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829201.789463:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789468:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829201.789474:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201]  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829201.789482:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829201.789484:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829201.789486:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829201.789489:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829201.789491:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829201.789493:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829201.789495:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829201.789501:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789505:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829201.789507:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789508:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.789510:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829201.789512:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789517:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829201.789520:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789525:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829201.789527:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789528:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.789529:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789531:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829201.789532:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829201.789534:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829201.789536:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829201.789538:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829201.789540:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829201.789545:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327320448:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829201.789549:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789551:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.789553:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327320448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789557:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829201.789559:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829201.789561:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829201.789563:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829201.789564:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829201.789565:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829201.789566:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829201.789568:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:1.0:1787829201.789571:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829201.789573:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829201.789574:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829201.789576:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829201.789577:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789578:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789583:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829201.789585:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789586:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829201.789587:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829201.789588:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829201.789591:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789596:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829201.789597:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.789598:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829201.789599:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829201.789601:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829201.789603:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320576:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829201.789605:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829201.789606:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829201.789607:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.789609:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829201.789611:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829201.789612:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829201.789615:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829201.789616:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829201.789625:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829201.789628:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320576, portal 10
00000100:00000001:1.0:1787829201.789630:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829201.789631:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829201.789635:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789640:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829201.789641:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829201.789643:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829201.789645:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327320576, offset 0
00000400:00000200:1.0:1787829201.789647:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829201.789650:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.789653:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829201.789682:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829201.789684:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829201.789686:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829201.789687:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829201.789690:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40276:40276:256:4294967295] 192.168.203.116@tcp LPNI seq info [40276:40276:8:4294967295]
00000400:00000200:1.0:1787829201.789693:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.789697:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829201.789700:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.789703:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829201.789706:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829201.789708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829201.789712:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829201.789714:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829201.789716:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.789717:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829201.789720:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829201.789722:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829201.789724:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829201.789726:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829201.789727:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829201.789730:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789735:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.789737:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789742:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789743:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829201.789745:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829201.789748:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829201.789750:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829201.789751:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298021899
00000100:00000001:1.0:1787829201.789753:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298022572
00000100:00000001:1.0:1787829201.789755:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298021899
00000100:00000001:1.0:1787829201.789756:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298022572
00000100:00000001:1.0:1787829201.789757:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829201.789759:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789760:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829201.789762:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201]  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829201.789767:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829201.789768:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829201.789769:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829201.789771:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829201.789772:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829201.789773:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829201.789776:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829201.789781:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789785:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829201.789786:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789787:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.789788:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829201.789790:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789794:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829201.789796:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789800:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829201.789802:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789803:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.789803:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.789804:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829201.789806:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829201.789807:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829201.789809:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829201.789811:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829201.789813:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829201.789820:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320576:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829201.789823:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829201.789827:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789840:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829201.789841:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.789844:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829201.789846:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829201.789848:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829201.789851:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320704:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829201.789855:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829201.789857:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829201.789859:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.789861:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829201.789865:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829201.789867:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829201.789871:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829201.789873:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829201.789876:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829201.789878:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320704, portal 4
00000100:00000001:1.0:1787829201.789880:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829201.789881:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829201.789885:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789890:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829201.789891:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829201.789892:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829201.789894:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327320704, offset 0
00000400:00000200:1.0:1787829201.789897:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829201.789900:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.789902:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829201.789904:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829201.789906:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829201.789907:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829201.789909:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829201.789911:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40277:40277:256:4294967295] 192.168.203.116@tcp LPNI seq info [40277:40277:8:4294967295]
00000400:00000200:1.0:1787829201.789914:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.789919:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829201.789922:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.789924:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829201.789926:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829201.789928:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829201.789931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829201.789933:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829201.789935:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.789937:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829201.789939:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829201.789942:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829201.789944:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829201.789945:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829201.789947:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829201.789949:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789954:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.789956:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.789960:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789961:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829201.789963:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829201.789965:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829201.789967:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829201.789969:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298021906
00000100:00000001:1.0:1787829201.789970:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298022572
00000100:00000001:1.0:1787829201.789972:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298021906
00000100:00000001:1.0:1787829201.789974:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298022572
00000100:00000001:1.0:1787829201.789975:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829201.789977:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.789978:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829201.789980:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201]  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829201.789986:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829201.789987:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829201.789988:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829201.789989:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829201.790004:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829201.790008:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829201.790011:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829201.790016:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790021:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829201.790022:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790023:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.790024:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829201.790026:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790031:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829201.790033:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790037:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829201.790039:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790040:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.790040:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790041:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829201.790043:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829201.790044:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829201.790047:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829201.790048:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829201.790051:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829201.790056:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320704:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829201.790058:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829201.790060:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790064:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829201.790065:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.790066:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829201.790067:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829201.790068:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829201.790070:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320832:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829201.790072:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829201.790073:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829201.790075:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829201.790076:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829201.790078:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829201.790080:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829201.790082:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829201.790083:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829201.790085:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829201.790087:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320832, portal 4
00000100:00000001:1.0:1787829201.790089:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829201.790090:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829201.790094:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790098:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829201.790100:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829201.790102:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829201.790103:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327320832, offset 0
00000400:00000200:1.0:1787829201.790106:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829201.790108:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.790111:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829201.790113:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829201.790114:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829201.790116:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829201.790117:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829201.790119:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40278:40278:256:4294967295] 192.168.203.116@tcp LPNI seq info [40278:40278:8:4294967295]
00000400:00000200:1.0:1787829201.790122:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829201.790126:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829201.790129:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.790131:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829201.790133:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829201.790135:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829201.790138:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829201.790140:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829201.790142:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829201.790144:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829201.790146:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829201.790148:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829201.790150:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829201.790151:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829201.790152:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829201.790155:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790160:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.790162:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790166:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.790167:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829201.790169:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829201.790171:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829201.790173:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829201.790175:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298021911
00000100:00000001:1.0:1787829201.790176:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298022572
00000100:00000001:1.0:1787829201.790178:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298021911
00000100:00000001:1.0:1787829201.790179:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298022572
00000100:00000001:1.0:1787829201.790180:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829201.790182:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.790183:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829201.790185:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201]  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829201.790189:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829201.790190:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829201.790191:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829201.790193:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829201.790194:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829201.790195:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829201.790197:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829201.790202:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790206:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829201.790207:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790208:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.790209:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829201.790211:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790215:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829201.790217:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790222:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829201.790224:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790225:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829201.790226:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790227:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829201.790228:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829201.790230:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829201.790231:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829201.790232:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829201.790235:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829201.790239:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327320832:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829201.790241:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790243:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.790245:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327320576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790251:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829201.790253:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829201.790254:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829201.790257:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829201.790258:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829201.790259:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829201.790261:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829201.790263:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829201.790265:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829201.790267:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829201.790269:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829201.790271:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829201.790273:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790275:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.790279:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327320704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790287:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829201.790288:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829201.790290:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829201.790292:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829201.790294:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829201.790295:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829201.790297:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829201.790299:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:1.0:1787829201.790301:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829201.790302:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829201.790304:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829201.790305:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829201.790306:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790307:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829201.790310:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327320832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829201.790314:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829201.790315:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829201.790316:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829201.790318:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829201.790319:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829201.790319:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829201.790320:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829201.790321:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829201.790322:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829201.790323:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829201.790324:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829201.790325:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829201.790326:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790327:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829201.790329:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829201.790330:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829201.790331:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829201.790333:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829202.300787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829202.300795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829202.300799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829202.300804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829202.300808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829202.300812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829202.300818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829202.300821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829202.300823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829202.492741:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298022572, at 4298022592
00000100:00000040:3.0:1787829202.492741:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298022572, at 4298022592
00000100:00000010:3.0:1787829202.492760:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63b40.
00000100:00000010:2.0:1787829202.492765:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d80.
00000020:00000001:2.0:1787829202.620875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829202.620879:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829202.620881:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829203.324793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829203.324800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829203.324803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829203.324808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829203.324811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829203.324815:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829203.324818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829203.324820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829203.324824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829204.348804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829204.348811:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829204.348814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829204.348820:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829204.348823:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829204.348826:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829204.348829:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829204.348831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829204.348833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829204.668821:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829204.668825:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829204.668827:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829205.372783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829205.372791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829205.372794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829205.372800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829205.372804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829205.372808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829205.372810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829205.372813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829205.372816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829205.692801:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829205.692804:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829205.692807:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829206.396770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829206.396777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829206.396781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829206.396786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829206.396790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829206.396793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829206.396796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829206.396799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829206.396803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829206.908865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829206.908874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829206.908876:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829206.908879:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829206.908883:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829206.908889:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3354
00000400:00000200:2.0:1787829206.908896:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829206.908901:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829206.908904:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829206.908906:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.908908:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.908910:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.908912:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829206.908915:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829206.908916:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829206.908919:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829206.908921:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829206.908923:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829206.908924:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829206.908926:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.908927:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.908929:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.908930:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829206.908933:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829206.908935:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829206.908937:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829206.908940:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829206.908943:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829206.908945:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.908947:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829206.908949:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829206.908974:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787829206.908978:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829206.908993:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829206.908994:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.908996:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829206.908998:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829206.909002:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829206.909008:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829206.909014:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829206.909021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829206.909029:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829206.909032:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829206.909036:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829206.909054:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829206.909056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829206.909059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829206.909060:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829206.909061:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829206.909064:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829206.909065:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3354
00000400:00000200:2.0:1787829206.909068:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829206.909069:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829206.909071:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829206.909073:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909075:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.909076:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909078:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829206.909080:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829206.909082:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829206.909083:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829206.909084:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829206.909085:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829206.909087:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829206.909089:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909091:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.909092:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909104:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829206.909106:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829206.909107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909109:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829206.909110:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829206.909114:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829206.909115:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909117:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829206.909121:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829206.909123:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909124:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829206.909129:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829206.909131:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829206.909133:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829206.909134:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909135:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829206.909136:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829206.909137:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829206.909139:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:1.0:1787829206.909144:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829206.909144:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.909156:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00100000:2.0:1787829206.909157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829206.909161:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787829206.909171:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829206.909175:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829206.909180:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829206.909182:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829206.909211:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829206.909220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829206.909233:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787829206.909234:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829206.909242:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909243:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829206.909251:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:1.0:1787829206.909255:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787829206.909256:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829206.909281:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3354
02000000:00000001:1.0:1787829206.909283:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829206.909291:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829206.909298:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:1.0:1787829206.909299:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327320960:192.168.203.116@tcp:250:kworker.0
00000400:00000200:2.0:1787829206.909310:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829206.909312:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787829206.909317:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000001:1.0:1787829206.909319:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829206.909325:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829206.909330:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:1.0:1787829206.909331:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787829206.909349:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.909351:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000010:1.0:1787829206.909360:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:2.0:1787829206.909361:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:1.0:1787829206.909371:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829206.909371:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829206.909375:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829206.909379:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829206.909381:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829206.909383:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829206.909385:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829206.909394:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829206.909398:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909402:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.909404:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829206.909408:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000100:00000001:2.0:1787829206.909408:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829206.909421:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829206.909424:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909426:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829206.909427:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829206.909430:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829206.909441:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909443:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787829206.909446:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000008:00100000:2.0:1787829206.909446:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829206.909454:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909456:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829206.909460:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327320960, portal 25
00000100:00000010:2.0:1787829206.909461:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000100:00000001:1.0:1787829206.909471:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000040:2.0:1787829206.909471:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:1.0:1787829206.909476:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402430400 : -116696307121216 : ffff95dd861143c0)
02000000:00000001:2.0:1787829206.909482:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829206.909484:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909490:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1787829206.909492:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829206.909494:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829206.909510:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829206.909514:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:1.0:1787829206.909527:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787829206.909527:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829206.909535:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829206.909539:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00100000:2.0:1787829206.909544:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787829206.909547:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327320960, offset 0
00000400:00000200:1.0:1787829206.909562:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909567:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829206.909572:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829206.909577:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829206.909588:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829206.909590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829206.909593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787829206.909594:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829206.909596:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829206.909599:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829206.909603:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829206.909607:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3354
00000400:00000200:2.0:1787829206.909612:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.909614:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829206.909622:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829206.909626:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829206.909629:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829206.909632:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787829206.909633:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.909645:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829206.909651:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787829206.909652:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829206.909686:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829206.909687:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829206.909688:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829206.909690:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829206.909692:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829206.909695:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829206.909701:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909703:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829206.909707:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829206.909710:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829206.909712:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829206.909717:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829206.909723:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829206.909725:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829206.909728:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:2.0:1787829206.909739:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829206.909742:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40279:40279:256:4294967295] 192.168.203.116@tcp LPNI seq info [40279:40279:8:4294967295]
02000000:00000001:2.0:1787829206.909748:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787829206.909750:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829206.909759:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909763:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829206.909769:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787829206.909770:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000008:00000001:2.0:1787829206.909785:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829206.909804:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829206.909807:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000010:2.0:1787829206.909811:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000800:00000010:1.0:1787829206.909814:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000020:00000040:2.0:1787829206.909818:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000800:00000200:1.0:1787829206.909827:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829206.909828:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829206.909836:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829206.909837:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787829206.909846:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.1:1787829206.909853:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829206.909854:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829206.909872:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829206.909876:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
02000000:00000010:2.0:1787829206.909882:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000800:00000100:1.0:1787829206.909883:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.909893:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000001:2.0:1787829206.909894:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829206.909903:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:2.0:1787829206.909906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829206.909915:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829206.909928:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000100:00000001:2.0:1787829206.909931:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787829206.909932:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:2.0:1787829206.909942:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.909951:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787829206.909952:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829206.909978:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829206.909984:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829206.910007:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3364)
00000100:00000001:1.0:1787829206.910013:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.910019:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910026:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910028:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829206.910035:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829206.910038:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829206.910041:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298027693
00000100:00000010:1.0:1787829206.910046:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63f00.
00000100:00000001:1.0:1787829206.910053:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829206.910059:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298027693
00000100:00000010:1.0:1787829206.910062:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63a20.
00000100:00000001:1.0:1787829206.910065:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829206.910067:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829206.910070:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910072:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829206.910076:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206]  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829206.910083:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829206.910084:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829206.910086:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829206.910089:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829206.910090:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829206.910092:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829206.910112:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829206.910121:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910127:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829206.910129:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910131:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.910133:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829206.910137:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910143:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829206.910148:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910154:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829206.910157:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910159:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.910160:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910163:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829206.910166:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829206.910170:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829206.910174:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829206.910177:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829206.910181:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829206.910188:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327320960:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829206.910193:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910195:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.910198:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861143c0 x1874671327320960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910203:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829206.910206:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829206.910209:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829206.910212:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829206.910215:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829206.910217:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829206.910220:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829206.910223:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:1.0:1787829206.910227:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829206.910229:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829206.910231:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861143c0.
00000100:00000001:1.0:1787829206.910234:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829206.910235:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910238:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910246:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829206.910248:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910250:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829206.910252:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829206.910254:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829206.910257:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910271:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829206.910272:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829206.910275:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829206.910277:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829206.910279:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829206.910281:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321088:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829206.910300:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829206.910301:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829206.910302:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829206.910304:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829206.910306:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829206.910307:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829206.910310:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829206.910313:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829206.910316:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829206.910318:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321088, portal 10
00000100:00000001:1.0:1787829206.910320:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829206.910321:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:1.0:1787829206.910326:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910331:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829206.910333:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829206.910334:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829206.910336:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327321088, offset 0
00000400:00000200:1.0:1787829206.910339:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829206.910342:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.910345:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829206.910347:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829206.910349:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829206.910351:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829206.910353:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829206.910357:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40280:40280:256:4294967295] 192.168.203.116@tcp LPNI seq info [40280:40280:8:4294967295]
00000400:00000200:1.0:1787829206.910362:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.910369:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829206.910374:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.910378:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829206.910383:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829206.910386:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829206.910391:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829206.910394:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829206.910398:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.910401:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829206.910404:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829206.910408:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829206.910410:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829206.910411:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829206.910413:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829206.910416:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910422:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.910424:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910429:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910430:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829206.910433:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829206.910436:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829206.910439:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829206.910440:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298027019
00000100:00000001:1.0:1787829206.910442:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298027693
00000100:00000001:1.0:1787829206.910444:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298027019
00000100:00000001:1.0:1787829206.910446:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298027693
00000100:00000001:1.0:1787829206.910447:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829206.910449:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910451:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829206.910453:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206]  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829206.910459:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829206.910460:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829206.910461:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829206.910462:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829206.910464:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829206.910482:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829206.910485:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829206.910493:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910499:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829206.910500:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910502:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.910503:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829206.910506:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910512:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829206.910515:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910521:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829206.910523:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910525:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.910526:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910528:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829206.910531:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829206.910533:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829206.910537:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829206.910539:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829206.910543:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829206.910550:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321088:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829206.910553:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829206.910557:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910562:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829206.910564:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829206.910567:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829206.910569:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829206.910572:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829206.910576:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321216:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829206.910581:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829206.910584:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829206.910587:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829206.910590:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829206.910595:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829206.910599:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829206.910604:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829206.910608:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829206.910613:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829206.910617:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321216, portal 4
00000100:00000001:1.0:1787829206.910621:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829206.910623:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:1.0:1787829206.910632:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910642:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829206.910645:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829206.910647:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829206.910650:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327321216, offset 0
00000400:00000200:1.0:1787829206.910655:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829206.910659:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.910663:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829206.910666:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829206.910668:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829206.910671:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829206.910673:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829206.910676:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40281:40281:256:4294967295] 192.168.203.116@tcp LPNI seq info [40281:40281:8:4294967295]
00000400:00000200:1.0:1787829206.910680:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.910686:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829206.910690:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.910693:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829206.910697:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829206.910700:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829206.910704:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829206.910706:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829206.910710:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.910712:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829206.910739:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829206.910743:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829206.910746:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829206.910748:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829206.910751:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829206.910754:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910762:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.910765:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910787:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910789:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829206.910792:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829206.910794:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829206.910797:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829206.910799:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298027027
00000100:00000001:1.0:1787829206.910801:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298027693
00000100:00000001:1.0:1787829206.910802:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298027027
00000100:00000001:1.0:1787829206.910804:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298027693
00000100:00000001:1.0:1787829206.910805:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829206.910807:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.910808:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829206.910810:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206]  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829206.910815:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829206.910816:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829206.910818:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829206.910819:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829206.910821:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829206.910822:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829206.910824:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829206.910830:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910834:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829206.910835:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910855:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.910856:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829206.910859:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910865:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829206.910868:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910875:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829206.910877:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910879:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.910880:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.910882:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829206.910883:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829206.910885:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829206.910888:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829206.910890:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829206.910893:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829206.910899:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321216:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829206.910903:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829206.910906:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910912:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829206.910913:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829206.910915:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829206.910917:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829206.910919:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829206.910921:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321344:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829206.910924:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829206.910926:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829206.910927:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829206.910929:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829206.910933:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829206.910935:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829206.910938:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829206.910941:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829206.910944:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829206.910947:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321344, portal 4
00000100:00000001:1.0:1787829206.910949:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829206.910951:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:1.0:1787829206.910977:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.910984:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829206.910987:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829206.910989:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829206.910993:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327321344, offset 0
00000400:00000200:1.0:1787829206.911013:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829206.911017:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.911019:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829206.911021:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829206.911023:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829206.911025:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829206.911027:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829206.911029:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40282:40282:256:4294967295] 192.168.203.116@tcp LPNI seq info [40282:40282:8:4294967295]
00000400:00000200:1.0:1787829206.911032:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829206.911036:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829206.911039:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.911041:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829206.911044:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829206.911046:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829206.911049:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829206.911051:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829206.911054:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829206.911055:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829206.911057:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829206.911060:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829206.911062:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829206.911063:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829206.911065:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829206.911068:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911074:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.911076:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911080:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.911081:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829206.911083:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829206.911086:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829206.911088:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829206.911089:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298027032
00000100:00000001:1.0:1787829206.911091:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298027693
00000100:00000001:1.0:1787829206.911093:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298027032
00000100:00000001:1.0:1787829206.911111:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298027693
00000100:00000001:1.0:1787829206.911113:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829206.911115:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.911116:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829206.911120:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206]  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829206.911127:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829206.911128:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829206.911130:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829206.911132:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829206.911134:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829206.911136:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829206.911138:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829206.911146:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911158:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829206.911159:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911161:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.911163:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829206.911166:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911171:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829206.911175:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911180:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829206.911182:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911183:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829206.911185:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911187:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829206.911188:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829206.911191:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829206.911194:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829206.911196:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829206.911199:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829206.911206:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321344:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829206.911209:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911211:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.911213:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327321088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911219:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829206.911221:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829206.911223:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829206.911225:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829206.911227:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829206.911229:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829206.911231:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829206.911232:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:1.0:1787829206.911235:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829206.911237:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829206.911238:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:1.0:1787829206.911241:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829206.911243:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911244:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.911247:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327321216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911253:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829206.911254:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829206.911256:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829206.911258:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829206.911258:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829206.911259:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829206.911261:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829206.911262:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:1.0:1787829206.911264:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829206.911265:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829206.911283:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:1.0:1787829206.911284:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829206.911285:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911286:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829206.911288:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327321344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829206.911291:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829206.911292:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829206.911293:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829206.911295:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829206.911295:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829206.911296:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829206.911298:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829206.911299:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:1.0:1787829206.911300:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829206.911301:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829206.911302:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:1.0:1787829206.911304:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829206.911304:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911305:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829206.911308:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829206.911309:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829206.911310:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829206.911311:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829207.420794:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829207.420802:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829207.420804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829207.420811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829207.420816:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829207.420819:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829207.420822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829207.420826:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829207.420829:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829207.612754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298027693, at 4298027712
00000100:00000040:3.0:1787829207.612754:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298027693, at 4298027712
00000100:00000010:2.0:1787829207.612769:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f00.
00000100:00000010:3.0:1787829207.612780:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a20.
00000020:00000001:2.0:1787829207.740836:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829207.740841:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829207.740846:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829208.444722:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829208.444728:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829208.444729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829208.444733:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829208.444736:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829208.444738:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829208.444740:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829208.444742:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829208.444744:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829208.764848:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829208.764852:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829208.764855:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829209.468750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829209.468756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829209.468758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829209.468763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829209.468767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829209.468771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829209.468774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829209.468777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829209.468780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829210.492780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829210.492785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829210.492786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829210.492790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829210.492793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829210.492796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829210.492798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829210.492800:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829210.492802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829210.812847:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829210.812851:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829210.812854:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829211.516784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829211.516792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829211.516795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829211.516800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829211.516804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829211.516808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829211.516811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829211.516814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829211.516817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829211.836792:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829211.836795:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829211.836798:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829212.029018:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829212.029024:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829212.029029:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829212.029039:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029052:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829212.029056:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.029060:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829212.029064:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829212.029071:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829212.029078:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327321472:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829212.029085:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829212.029090:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829212.029093:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.029099:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829212.029104:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829212.029108:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829212.029117:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829212.029120:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829212.029127:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829212.029134:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321472, portal 25
00000100:00000001:1.0:1787829212.029139:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829212.029141:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829212.029151:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029161:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829212.029165:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829212.029170:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829212.029175:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327321472, offset 0
00000400:00000200:1.0:1787829212.029187:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829212.029201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.029215:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829212.029225:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829212.029236:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829212.029244:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829212.029251:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829212.029260:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40283:40283:256:4294967295] 192.168.203.116@tcp LPNI seq info [40283:40283:8:4294967295]
00000400:00000200:1.0:1787829212.029268:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.029296:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829212.029310:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.029317:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829212.029327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829212.029333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829212.029343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829212.029350:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829212.029365:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.029368:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829212.029377:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829212.029385:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829212.029391:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829212.029395:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829212.029401:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829212.029409:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029440:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.029449:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029462:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.029465:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829212.029472:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829212.029478:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829212.029482:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298032812
00000100:00000010:1.0:1787829212.029488:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63ea0.
00000100:00000001:1.0:1787829212.029505:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829212.029511:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298032812
00000100:00000010:1.0:1787829212.029516:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e635a0.
00000100:00000001:1.0:1787829212.029521:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829212.029534:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829212.029539:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.029545:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829212.029552:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829212/real 1787829212]  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829212.029574:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829212.029577:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829212.029580:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829212.029583:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829212.029587:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829212.029589:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829212.029593:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829212.029600:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029605:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829212.029606:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029608:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.029609:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829212.029611:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029616:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829212.029619:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029624:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829212.029626:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029627:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.029628:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029629:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829212.029631:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829212.029633:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829212.029635:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829212.029637:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829212.029639:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829212.029644:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327321472:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829212.029647:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029649:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.029651:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327321472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029697:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829212.029700:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829212.029702:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829212.029704:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829212.029706:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829212.029707:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829212.029708:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829212.029710:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829212.029713:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829212.029715:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829212.029716:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829212.029719:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829212.029719:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029721:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029726:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829212.029728:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.029729:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829212.029730:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829212.029731:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829212.029734:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029739:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829212.029740:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.029742:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829212.029753:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829212.029755:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829212.029756:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321600:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829212.029759:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829212.029760:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829212.029768:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.029769:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829212.029771:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829212.029773:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829212.029775:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829212.029777:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829212.029780:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829212.029782:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321600, portal 10
00000100:00000001:1.0:1787829212.029783:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829212.029784:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829212.029789:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029794:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829212.029795:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829212.029797:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829212.029799:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327321600, offset 0
00000400:00000200:1.0:1787829212.029802:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829212.029805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.029807:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829212.029819:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829212.029821:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829212.029823:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829212.029825:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829212.029827:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40284:40284:256:4294967295] 192.168.203.116@tcp LPNI seq info [40284:40284:8:4294967295]
00000400:00000200:1.0:1787829212.029830:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.029834:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829212.029837:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.029839:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829212.029842:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829212.029844:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829212.029847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829212.029849:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829212.029863:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.029865:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829212.029867:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829212.029869:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829212.029872:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829212.029873:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829212.029875:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829212.029878:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029883:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.029885:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029890:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.029891:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829212.029893:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829212.029896:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829212.029898:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829212.029900:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298032139
00000100:00000001:1.0:1787829212.029902:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298032812
00000100:00000001:1.0:1787829212.029903:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298032139
00000100:00000001:1.0:1787829212.029905:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298032812
00000100:00000001:1.0:1787829212.029906:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829212.029908:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.029909:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829212.029911:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829212/real 1787829212]  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829212.029916:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829212.029917:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829212.029918:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829212.029933:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829212.029934:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829212.029936:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829212.029938:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829212.029944:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029948:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829212.029949:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029950:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.029951:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829212.029953:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029967:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829212.029969:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.029974:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829212.029976:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029977:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.029978:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.029979:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829212.029981:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829212.029982:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829212.029984:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829212.029986:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829212.029988:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829212.029993:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321600:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829212.029995:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829212.029998:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030002:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829212.030003:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.030004:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829212.030006:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829212.030007:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829212.030009:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321728:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829212.030011:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829212.030012:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829212.030013:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.030015:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829212.030017:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829212.030018:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829212.030021:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829212.030023:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829212.030025:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829212.030027:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321728, portal 4
00000100:00000001:1.0:1787829212.030028:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829212.030029:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829212.030034:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030038:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829212.030040:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829212.030041:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829212.030043:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327321728, offset 0
00000400:00000200:1.0:1787829212.030045:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829212.030047:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.030050:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829212.030052:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829212.030054:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829212.030055:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829212.030057:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829212.030059:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40285:40285:256:4294967295] 192.168.203.116@tcp LPNI seq info [40285:40285:8:4294967295]
00000400:00000200:1.0:1787829212.030062:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.030066:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829212.030069:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.030071:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829212.030074:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829212.030075:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829212.030078:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829212.030080:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829212.030083:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.030084:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829212.030086:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829212.030088:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829212.030090:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829212.030091:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829212.030093:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829212.030095:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030100:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.030102:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030106:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.030107:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829212.030109:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829212.030111:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829212.030113:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829212.030115:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298032146
00000100:00000001:1.0:1787829212.030116:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298032812
00000100:00000001:1.0:1787829212.030118:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298032146
00000100:00000001:1.0:1787829212.030119:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298032812
00000100:00000001:1.0:1787829212.030120:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829212.030122:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.030123:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829212.030125:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829212/real 1787829212]  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829212.030129:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829212.030130:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829212.030131:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829212.030133:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829212.030134:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829212.030135:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829212.030137:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829212.030143:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030162:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829212.030163:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030164:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.030165:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829212.030167:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030171:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829212.030173:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030177:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829212.030179:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030180:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.030180:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030181:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829212.030182:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829212.030184:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829212.030187:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829212.030189:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829212.030193:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829212.030201:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321728:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829212.030204:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829212.030207:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030214:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829212.030216:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.030219:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829212.030220:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829212.030223:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829212.030226:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321856:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829212.030230:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829212.030231:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829212.030233:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829212.030235:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829212.030238:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829212.030240:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829212.030243:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829212.030245:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829212.030248:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829212.030250:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321856, portal 4
00000100:00000001:1.0:1787829212.030251:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829212.030252:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829212.030257:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030261:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829212.030263:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829212.030264:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829212.030267:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327321856, offset 0
00000400:00000200:1.0:1787829212.030270:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829212.030274:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.030278:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829212.030281:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829212.030283:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829212.030286:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829212.030288:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829212.030292:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40286:40286:256:4294967295] 192.168.203.116@tcp LPNI seq info [40286:40286:8:4294967295]
00000400:00000200:1.0:1787829212.030296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829212.030302:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829212.030308:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.030311:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829212.030314:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829212.030316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829212.030319:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829212.030321:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829212.030323:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829212.030325:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829212.030327:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829212.030329:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829212.030331:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829212.030332:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829212.030334:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829212.030337:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030342:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.030344:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030348:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.030349:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829212.030351:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829212.030353:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829212.030367:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829212.030368:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298032151
00000100:00000001:1.0:1787829212.030370:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298032812
00000100:00000001:1.0:1787829212.030371:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298032151
00000100:00000001:1.0:1787829212.030372:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298032812
00000100:00000001:1.0:1787829212.030374:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829212.030375:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.030376:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829212.030379:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829212/real 1787829212]  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829212.030384:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829212.030385:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829212.030386:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829212.030388:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829212.030388:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829212.030390:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829212.030392:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829212.030397:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030401:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829212.030402:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030403:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.030404:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829212.030406:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030410:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829212.030412:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030416:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829212.030418:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030419:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829212.030420:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030421:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829212.030431:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829212.030432:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829212.030434:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829212.030436:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829212.030439:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829212.030444:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327321856:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829212.030446:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030447:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.030449:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327321600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030453:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829212.030454:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829212.030455:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829212.030457:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829212.030458:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829212.030459:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829212.030460:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829212.030461:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829212.030462:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829212.030463:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829212.030464:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829212.030466:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829212.030467:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030468:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.030470:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327321728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030474:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829212.030474:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829212.030475:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829212.030477:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829212.030477:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829212.030478:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829212.030479:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829212.030480:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829212.030481:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829212.030482:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829212.030483:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829212.030485:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829212.030485:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030486:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829212.030489:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327321856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829212.030492:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829212.030493:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829212.030494:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829212.030495:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829212.030496:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829212.030497:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829212.030498:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829212.030499:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:1.0:1787829212.030500:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829212.030501:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829212.030501:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829212.030502:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829212.030503:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030504:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829212.030507:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829212.030508:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829212.030509:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829212.030510:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829212.540743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829212.540748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829212.540749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829212.540753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829212.540756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829212.540759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829212.540761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829212.540763:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829212.540765:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829212.732713:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298032812, at 4298032832
00000100:00000010:3.0:1787829212.732727:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e635a0.
00000100:00000040:2.0:1787829212.732729:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298032812, at 4298032832
00000100:00000010:2.0:1787829212.732736:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ea0.
00000100:00000001:2.0:1787829213.564767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829213.564773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829213.564775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829213.564779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829213.564782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829213.564785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829213.564786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829213.564788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829213.564791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829213.884845:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829213.884849:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829213.884853:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829214.588732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829214.588737:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829214.588738:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829214.588742:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829214.588764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829214.588768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829214.588770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829214.588772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829214.588775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829214.908848:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829214.908854:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829214.908860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829215.612743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829215.612752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829215.612754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829215.612759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829215.612763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829215.612766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829215.612768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829215.612771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829215.612773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829216.636721:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829216.636728:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829216.636730:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829216.636735:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829216.636738:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829216.636741:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829216.636744:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829216.636747:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829216.636749:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829216.956811:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829216.956816:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829216.956820:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829217.148754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829217.148764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829217.148767:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829217.148771:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829217.148776:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829217.148783:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3364
00000400:00000200:3.0:1787829217.148792:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829217.148799:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829217.148802:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829217.148805:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148808:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.148810:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148812:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829217.148816:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829217.148817:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829217.148820:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829217.148823:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829217.148825:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829217.148826:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829217.148828:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148830:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.148831:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148833:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829217.148835:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829217.148838:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148840:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829217.148842:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829217.148845:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829217.148848:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.148849:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829217.148851:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829217.148862:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829217.148865:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829217.148869:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829217.148870:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.148873:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829217.148876:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829217.148881:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829217.148902:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829217.148909:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829217.148919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829217.148928:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829217.148932:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829217.148936:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829217.148957:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829217.148959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829217.148962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829217.148963:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829217.148964:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829217.148966:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829217.148968:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3364
00000400:00000200:3.0:1787829217.148971:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829217.148973:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829217.148975:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829217.148977:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148978:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.148980:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.148982:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829217.148984:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829217.148987:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829217.148989:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829217.148992:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829217.148995:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829217.148998:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829217.149002:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149004:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829217.149006:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787829217.149007:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149012:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829217.149015:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829217.149018:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829217.149019:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829217.149024:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829217.149024:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829217.149029:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787829217.149034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829217.149038:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829217.149043:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149048:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829217.149050:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829217.149053:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787829217.149054:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829217.149057:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1787829217.149058:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149061:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.149062:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829217.149065:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829217.149065:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829217.149071:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00100000:1.0:1787829217.149072:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327321984:192.168.203.116@tcp:250:kworker.0
02000000:00000001:3.0:1787829217.149074:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829217.149077:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149081:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829217.149081:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829217.149084:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829217.149086:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.149087:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829217.149090:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829217.149091:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
02000000:00000001:1.0:1787829217.149093:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829217.149096:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829217.149100:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
00000100:00100000:3.0:1787829217.149104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787829217.149105:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149114:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829217.149115:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829217.149118:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829217.149121:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000100:00000040:3.0:1787829217.149123:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787829217.149129:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000001:3.0:1787829217.149139:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829217.149140:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327321984, portal 25
00000100:00080000:3.0:1787829217.149142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829217.149146:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1787829217.149147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829217.149149:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000001:3.0:1787829217.149149:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829217.149152:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829217.149156:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1787829217.149158:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829217.149161:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3364
00000400:00000200:3.0:1787829217.149170:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829217.149173:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1787829217.149174:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000040:1.0:1787829217.149179:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829217.149179:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1787829217.149182:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000040:3.0:1787829217.149186:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:1.0:1787829217.149188:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327321984, offset 0
00000100:00000001:3.0:1787829217.149189:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.149193:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149195:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829217.149197:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829217.149199:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829217.149201:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787829217.149203:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:3.0:1787829217.149203:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829217.149208:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829217.149209:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829217.149212:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149213:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.149214:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149216:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829217.149220:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:3.0:1787829217.149221:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829217.149225:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149227:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829217.149229:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829217.149231:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787829217.149234:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:3.0:1787829217.149234:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149237:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829217.149241:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829217.149243:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149244:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.149246:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:3.0:1787829217.149248:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829217.149251:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:1.0:1787829217.149254:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:3.0:1787829217.149255:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829217.149257:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149260:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829217.149261:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787829217.149264:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787829217.149264:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829217.149268:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000400:00000200:1.0:1787829217.149269:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149273:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.149278:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40287:40287:256:4294967295] 192.168.203.116@tcp LPNI seq info [40287:40287:8:4294967295]
00000100:00100000:3.0:1787829217.149281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829217.149289:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829217.149294:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829217.149296:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829217.149299:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829217.149311:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.149312:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1787829217.149313:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829217.149320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829217.149323:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829217.149324:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829217.149327:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1787829217.149329:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.149338:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00080000:3.0:1787829217.149338:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3364
00000400:00000200:3.0:1787829217.149343:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829217.149347:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:1.0:1787829217.149352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829217.149352:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1787829217.149360:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000040:3.0:1787829217.149360:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829217.149366:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.149371:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829217.149372:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829217.149374:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829217.149378:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829217.149379:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000200:1.1:1787829217.149379:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000020:00000001:3.0:1787829217.149382:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829217.149385:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000100:1.0:1787829217.149389:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829217.149389:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000010:1.0:1787829217.149393:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000020:00000001:3.0:1787829217.149394:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829217.149401:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829217.149403:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787829217.149404:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829217.149409:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829217.149416:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787829217.149416:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829217.149422:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:1.0:1787829217.149426:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000100:00080000:3.0:1787829217.149427:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000010:1.0:1787829217.149432:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:3.0:1787829217.149433:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829217.149438:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829217.149440:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787829217.149442:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829217.149446:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829217.149450:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829217.149451:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829217.149454:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829217.149460:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149464:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829217.149469:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829217.149470:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:3.0:1787829217.149472:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000040:1.0:1787829217.149475:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829217.149477:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829217.149479:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149484:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829217.149486:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829217.149489:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149490:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149493:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:3.0:1787829217.149495:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:1.0:1787829217.149500:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829217.149502:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.149507:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:3.0:1787829217.149512:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829217.149518:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298037932
00000100:00000001:3.0:1787829217.149525:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:1.0:1787829217.149526:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63420.
00000100:00000001:3.0:1787829217.149528:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149535:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787829217.149535:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149544:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298037932
00000100:00000001:3.0:1787829217.149545:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829217.149550:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63de0.
00000100:00000040:3.0:1787829217.149552:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3374)
00000100:00000001:1.0:1787829217.149554:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829217.149559:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829217.149561:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149564:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829217.149569:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829217/real 1787829217]  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829217.149578:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829217.149580:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829217.149582:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829217.149585:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829217.149587:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829217.149589:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829217.149591:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829217.149597:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149601:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829217.149603:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149604:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.149606:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829217.149608:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149613:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829217.149616:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149621:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829217.149624:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149625:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.149626:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149628:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829217.149629:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829217.149631:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829217.149634:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829217.149635:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829217.149638:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829217.149643:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327321984:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829217.149650:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149652:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.149654:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327321984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149694:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829217.149697:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829217.149699:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829217.149702:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829217.149704:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829217.149705:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829217.149707:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829217.149710:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:1.0:1787829217.149714:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829217.149716:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829217.149717:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:1.0:1787829217.149719:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829217.149720:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149722:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149729:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829217.149731:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149732:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829217.149734:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829217.149735:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829217.149737:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149742:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829217.149744:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.149745:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829217.149747:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829217.149748:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829217.149750:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322112:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829217.149752:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829217.149754:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829217.149755:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.149757:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829217.149759:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829217.149761:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.149763:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829217.149765:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829217.149768:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829217.149770:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322112, portal 10
00000100:00000001:1.0:1787829217.149771:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829217.149772:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:1.0:1787829217.149778:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149783:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829217.149784:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829217.149785:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829217.149787:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327322112, offset 0
00000400:00000200:1.0:1787829217.149790:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829217.149792:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829217.149795:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829217.149797:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829217.149799:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829217.149801:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829217.149803:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829217.149805:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40288:40288:256:4294967295] 192.168.203.116@tcp LPNI seq info [40288:40288:8:4294967295]
00000400:00000200:1.0:1787829217.149808:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829217.149813:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829217.149816:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.149818:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829217.149821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829217.149823:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829217.149826:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829217.149828:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829217.149831:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.149833:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829217.149835:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829217.149837:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829217.149840:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829217.149841:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829217.149842:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829217.149845:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149850:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.149852:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149857:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149858:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829217.149860:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829217.149863:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829217.149865:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829217.149866:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298037259
00000100:00000001:1.0:1787829217.149869:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298037932
00000100:00000001:1.0:1787829217.149870:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298037259
00000100:00000001:1.0:1787829217.149872:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298037932
00000100:00000001:1.0:1787829217.149874:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829217.149875:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.149876:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829217.149878:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829217/real 1787829217]  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829217.149900:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829217.149902:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829217.149904:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829217.149906:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829217.149908:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829217.149910:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829217.149913:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829217.149919:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149923:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829217.149924:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149925:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.149926:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829217.149929:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149934:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829217.149936:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149940:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829217.149942:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149943:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.149943:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.149945:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829217.149946:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829217.149948:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829217.149950:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829217.149951:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829217.149953:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829217.149958:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322112:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829217.149961:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829217.149963:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.149972:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829217.149973:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.149975:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829217.149977:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829217.149978:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829217.149979:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322240:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829217.149981:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829217.149983:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829217.149984:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.149985:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829217.149987:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829217.149988:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.149991:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829217.149992:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829217.149995:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829217.149997:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322240, portal 4
00000100:00000001:1.0:1787829217.149998:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829217.149999:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:1.0:1787829217.150004:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150008:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829217.150010:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829217.150011:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829217.150013:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327322240, offset 0
00000400:00000200:1.0:1787829217.150015:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829217.150018:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829217.150020:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829217.150022:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829217.150024:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829217.150026:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829217.150027:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829217.150030:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40289:40289:256:4294967295] 192.168.203.116@tcp LPNI seq info [40289:40289:8:4294967295]
00000400:00000200:1.0:1787829217.150033:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829217.150037:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829217.150041:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.150043:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829217.150045:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829217.150048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829217.150050:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829217.150052:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829217.150055:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.150057:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829217.150059:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829217.150061:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829217.150063:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829217.150064:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829217.150066:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829217.150068:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150074:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.150076:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150080:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.150081:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829217.150082:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829217.150085:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829217.150086:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829217.150088:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298037266
00000100:00000001:1.0:1787829217.150090:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298037932
00000100:00000001:1.0:1787829217.150091:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298037266
00000100:00000001:1.0:1787829217.150093:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298037932
00000100:00000001:1.0:1787829217.150094:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829217.150095:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.150097:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829217.150099:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829217/real 1787829217]  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829217.150105:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829217.150105:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829217.150106:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829217.150108:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829217.150109:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829217.150110:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829217.150112:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829217.150118:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150121:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829217.150122:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150123:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.150124:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829217.150126:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150130:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829217.150132:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150136:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829217.150138:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150139:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.150139:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150140:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829217.150141:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829217.150143:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829217.150146:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829217.150147:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829217.150150:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829217.150155:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322240:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829217.150157:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829217.150159:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150163:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829217.150163:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.150165:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829217.150166:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829217.150167:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829217.150168:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322368:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829217.150170:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829217.150171:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829217.150172:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829217.150173:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829217.150175:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829217.150176:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829217.150178:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829217.150179:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829217.150181:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829217.150183:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322368, portal 4
00000100:00000001:1.0:1787829217.150184:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829217.150185:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:1.0:1787829217.150189:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150194:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829217.150196:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829217.150198:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00000200:1.0:1787829217.150201:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327322368, offset 0
00000400:00000200:1.0:1787829217.150205:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829217.150209:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829217.150212:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829217.150215:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829217.150217:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829217.150220:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829217.150222:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829217.150225:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40290:40290:256:4294967295] 192.168.203.116@tcp LPNI seq info [40290:40290:8:4294967295]
00000400:00000200:1.0:1787829217.150230:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829217.150237:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829217.150242:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.150246:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829217.150250:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829217.150253:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829217.150256:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829217.150258:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829217.150262:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829217.150263:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829217.150265:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829217.150268:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829217.150270:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:1.0:1787829217.150271:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:1.0:1787829217.150273:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829217.150276:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150281:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.150284:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150290:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.150292:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829217.150296:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829217.150299:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829217.150303:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829217.150305:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298037271
00000100:00000001:1.0:1787829217.150307:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298037932
00000100:00000001:1.0:1787829217.150310:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298037271
00000100:00000001:1.0:1787829217.150312:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298037932
00000100:00000001:1.0:1787829217.150315:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829217.150317:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.150319:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829217.150323:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829217/real 1787829217]  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829217.150333:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829217.150334:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829217.150337:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829217.150338:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829217.150340:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829217.150341:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829217.150344:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829217.150349:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150354:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829217.150355:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150357:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.150357:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829217.150360:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150364:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829217.150366:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150372:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829217.150373:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150374:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829217.150375:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150377:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829217.150378:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829217.150379:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829217.150381:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829217.150382:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829217.150385:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829217.150389:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322368:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829217.150391:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150392:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.150395:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327322112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150401:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829217.150403:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829217.150406:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829217.150408:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829217.150410:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829217.150412:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829217.150414:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829217.150416:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:1.0:1787829217.150418:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829217.150420:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829217.150422:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:1.0:1787829217.150425:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829217.150426:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150429:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.150433:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327322240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150441:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829217.150442:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829217.150444:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829217.150447:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829217.150448:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829217.150449:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829217.150450:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829217.150451:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:1.0:1787829217.150453:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829217.150454:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829217.150454:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:1.0:1787829217.150457:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829217.150458:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150459:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829217.150461:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327322368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829217.150466:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829217.150467:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829217.150468:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829217.150470:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829217.150470:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829217.150471:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829217.150472:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829217.150473:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:1.0:1787829217.150474:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829217.150475:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829217.150476:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829217.150477:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829217.150478:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150479:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829217.150481:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829217.150482:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829217.150483:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829217.150485:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829217.660814:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829217.660822:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829217.660825:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829217.660831:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829217.660834:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829217.660839:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829217.660841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829217.660844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829217.660846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829217.852787:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298037932, at 4298037952
00000100:00000040:3.0:1787829217.852787:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298037932, at 4298037952
00000100:00000010:3.0:1787829217.852805:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63de0.
00000100:00000010:2.0:1787829217.852807:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63420.
00000020:00000001:2.0:1787829217.980921:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829217.980938:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829217.980941:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829218.684776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829218.684784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829218.684786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829218.684791:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829218.684794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829218.684798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829218.684801:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829218.684803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829218.684806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829219.708722:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829219.708730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829219.708732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829219.708738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829219.708742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829219.708745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829219.708748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829219.708750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829219.708753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829220.028857:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829220.028862:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829220.028879:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829220.732799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829220.732807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829220.732810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829220.732815:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829220.732818:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829220.732822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829220.732825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829220.732828:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829220.732830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829221.052815:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829221.052818:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829221.052821:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829221.756718:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829221.756725:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829221.756729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829221.756734:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829221.756738:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829221.756742:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829221.756746:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829221.756749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829221.756753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829222.268794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829222.268809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829222.268812:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829222.268817:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829222.268823:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829222.268832:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3369
00000400:00000200:3.0:1787829222.268845:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829222.268867:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829222.268872:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829222.268876:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.268880:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829222.268882:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.268885:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829222.268891:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829222.268893:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829222.268897:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829222.268901:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829222.268904:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829222.268908:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829222.268913:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.268916:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829222.268918:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.268920:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829222.268924:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829222.268928:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829222.268931:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829222.268934:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829222.268939:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829222.268943:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.268945:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829222.268949:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829222.268961:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829222.268965:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829222.268970:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829222.268973:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.268977:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829222.268981:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829222.268986:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829222.268996:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829222.269004:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829222.269016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829222.269030:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829222.269035:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829222.269041:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829222.269071:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829222.269074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829222.269078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829222.269080:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829222.269082:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829222.269086:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829222.269089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3369
00000400:00000200:3.0:1787829222.269094:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829222.269097:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829222.269101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829222.269105:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269108:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829222.269111:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829222.269112:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787829222.269114:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829222.269118:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787829222.269120:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787829222.269122:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787829222.269124:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829222.269128:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787829222.269131:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787829222.269133:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829222.269139:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829222.269145:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829222.269147:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829222.269151:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829222.269154:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:1.0:1787829222.269156:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787829222.269158:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787829222.269160:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787829222.269164:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:1.0:1787829222.269165:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269170:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:1.0:1787829222.269173:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327322496:192.168.203.116@tcp:250:kworker.0
00000020:00000040:3.0:1787829222.269176:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:1.0:1787829222.269179:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787829222.269183:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829222.269185:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787829222.269188:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829222.269190:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829222.269193:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829222.269194:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829222.269201:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829222.269203:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829222.269203:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829222.269210:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269213:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787829222.269214:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829222.269220:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000008:00100000:3.0:1787829222.269222:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787829222.269226:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000008:00000001:3.0:1787829222.269228:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269232:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829222.269236:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322496, portal 25
00000100:00000010:3.0:1787829222.269239:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000100:00000001:1.0:1787829222.269242:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000040:3.0:1787829222.269244:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:1.0:1787829222.269246:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
02000000:00000001:3.0:1787829222.269250:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829222.269254:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269258:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1787829222.269259:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829222.269261:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829222.269265:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829222.269272:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:1.0:1787829222.269278:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787829222.269279:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829222.269285:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829222.269291:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98330.
00000100:00100000:3.0:1787829222.269291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829222.269302:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787829222.269305:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327322496, offset 0
00000100:00000001:3.0:1787829222.269308:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.269316:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:3.0:1787829222.269326:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829222.269330:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829222.269342:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829222.269345:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829222.269350:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829222.269351:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1787829222.269358:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829222.269360:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829222.269362:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829222.269365:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829222.269368:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:3.0:1787829222.269372:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3369
00000400:00000200:1.0:1787829222.269376:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787829222.269381:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829222.269382:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787829222.269391:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:1.0:1787829222.269392:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40291:40291:256:4294967295] 192.168.203.116@tcp LPNI seq info [40291:40291:8:4294967295]
00000100:00080000:3.0:1787829222.269398:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787829222.269402:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829222.269407:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269411:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829222.269415:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829222.269422:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787829222.269422:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829222.269427:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829222.269432:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:1.0:1787829222.269434:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829222.269436:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829222.269440:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000010:1.0:1787829222.269442:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000020:00000001:3.0:1787829222.269444:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829222.269449:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:1.0:1787829222.269453:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829222.269458:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829222.269464:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829222.269466:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829222.269472:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269475:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829222.269478:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829222.269481:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269484:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269487:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829222.269487:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829222.269491:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787829222.269496:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829222.269497:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269500:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298043052
00000100:00000001:3.0:1787829222.269502:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829222.269507:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000010:1.0:1787829222.269509:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e631e0.
00000008:00000001:3.0:1787829222.269513:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269517:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269518:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000010:3.0:1787829222.269524:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829222.269525:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298043052
00000020:00000040:3.0:1787829222.269532:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000010:1.0:1787829222.269533:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63840.
02000000:00000001:3.0:1787829222.269540:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829222.269541:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:3.0:1787829222.269545:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269546:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829222.269551:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787829222.269552:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829222.269556:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787829222.269558:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829222.269560:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269562:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1787829222.269570:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:1.0:1787829222.269573:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787829222.269576:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269578:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829222.269583:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00100000:3.0:1787829222.269585:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829222.269587:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829222.269590:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829222.269599:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787829222.269600:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829222.269606:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829222.269616:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829222.269618:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829222.269623:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829222.269628:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829222.269632:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829222.269635:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269636:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829222.269640:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322624:192.168.203.116@tcp:38:kworker.0
00000100:00080000:3.0:1787829222.269642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829222.269646:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787829222.269650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829222.269652:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829222.269679:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829222.269683:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829222.269687:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3369
00000400:00000200:3.0:1787829222.269692:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829222.269693:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829222.269698:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787829222.269698:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829222.269703:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000001:1.0:1787829222.269704:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829222.269711:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
00000100:00000040:3.0:1787829222.269711:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:1.0:1787829222.269718:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269719:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829222.269724:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269730:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829222.269731:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:3.0:1787829222.269736:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:1.0:1787829222.269737:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630d68.
00000020:00000040:3.0:1787829222.269740:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000010:1.0:1787829222.269743:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98a18.
00000020:00000001:3.0:1787829222.269745:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787829222.269747:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322624, portal 10
00000020:00000040:3.0:1787829222.269749:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829222.269750:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829222.269753:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000020:00000001:3.0:1787829222.269756:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829222.269761:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829222.269764:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829222.269769:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269775:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829222.269776:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787829222.269781:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829222.269782:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829222.269786:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98dd0.
00000100:00000001:3.0:1787829222.269788:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:1.0:1787829222.269790:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327322624, offset 0
00000020:00000040:3.0:1787829222.269793:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:1.0:1787829222.269798:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829222.269802:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829222.269805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829222.269809:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.269812:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:3.0:1787829222.269814:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829222.269817:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829222.269821:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:3.0:1787829222.269821:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787829222.269826:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787829222.269828:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.269830:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829222.269833:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787829222.269835:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40292:40292:255:4294967295] 192.168.203.116@tcp LPNI seq info [40292:40292:7:4294967295]
00000008:00100000:3.0:1787829222.269837:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829222.269843:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.269844:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829222.269862:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.269871:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000010:3.0:1787829222.269872:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829222.269878:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000800:00000200:1.0:1787829222.269883:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829222.269885:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829222.269889:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787829222.269893:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955d00.
00000100:00000001:3.0:1787829222.269893:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829222.269897:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829222.269900:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829222.269902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000010:3.0:1787829222.269908:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000800:00000200:1.1:1787829222.269910:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829222.269916:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269916:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829222.269925:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000100:00100000:3.0:1787829222.269928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829222.269933:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829222.269938:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298042379
00000100:00000001:1.0:1787829222.269945:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298043052
00000100:00000001:3.0:1787829222.269946:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829222.269951:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269952:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298042379
00000100:00000001:1.0:1787829222.269957:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298043052
00000100:00000040:3.0:1787829222.269959:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829222.269963:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829222.269969:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269977:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829222.269978:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.269983:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829222.269988:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3380)
00000100:00000001:1.0:1787829222.269990:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829222.269993:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829222.269997:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829222.269999:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829222.270001:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829222.270009:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829222.270018:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829222.270019:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829222.270022:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829222.270024:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829222.270026:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829222.270030:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322752:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829222.270034:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829222.270036:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829222.270038:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829222.270039:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829222.270043:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829222.270045:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.270051:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829222.270054:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd946304d0.
00000400:00000010:1.0:1787829222.270058:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829222.270061:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322752, portal 4
00000100:00000001:1.0:1787829222.270064:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829222.270065:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:1.0:1787829222.270072:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829222.270080:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829222.270082:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829222.270085:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98110.
00000100:00000200:1.0:1787829222.270087:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327322752, offset 0
00000400:00000200:1.0:1787829222.270092:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829222.270097:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829222.270100:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829222.270103:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829222.270106:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829222.270109:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829222.270111:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829222.270114:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40293:40293:254:4294967295] 192.168.203.116@tcp LPNI seq info [40293:40293:6:4294967295]
00000400:00000200:1.0:1787829222.270118:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829222.270124:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829222.270128:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829222.270132:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829222.270136:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829222.270140:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829222.270143:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829222.270145:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829222.270149:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829222.270151:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298042386
00000100:00000001:1.0:1787829222.270153:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298043052
00000100:00000001:1.0:1787829222.270155:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298042386
00000100:00000001:1.0:1787829222.270158:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298043052
00000100:00000001:1.0:1787829222.270160:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829222.270162:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.270165:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829222.270169:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829222.270178:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829222.270180:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829222.270182:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829222.270184:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829222.270187:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829222.270189:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322880:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829222.270193:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829222.270194:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829222.270196:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829222.270197:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829222.270201:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829222.270203:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829222.270206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829222.270209:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829222.270212:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829222.270216:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327322880, portal 4
00000100:00000001:1.0:1787829222.270218:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829222.270219:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:1.0:1787829222.270225:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829222.270232:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829222.270235:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829222.270238:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98990.
00000100:00000200:1.0:1787829222.270240:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327322880, offset 0
00000400:00000200:1.0:1787829222.270245:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829222.270249:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829222.270253:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829222.270256:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829222.270258:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829222.270260:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829222.270263:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829222.270266:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40294:40294:253:4294967295] 192.168.203.116@tcp LPNI seq info [40294:40294:5:4294967295]
00000400:00000200:1.0:1787829222.270271:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829222.270276:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829222.270280:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829222.270285:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955900.
00000800:00000200:1.0:1787829222.270288:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829222.270293:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829222.270295:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829222.270299:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829222.270303:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829222.270305:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298042391
00000100:00000001:1.0:1787829222.270307:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298043052
00000100:00000001:1.0:1787829222.270309:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298042391
00000100:00000001:1.0:1787829222.270312:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298043052
00000100:00000001:1.0:1787829222.270314:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829222.270316:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.270319:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.270321:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829222.270325:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829222.270327:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829222.270329:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829222.270330:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829222.270332:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829222.270334:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829222.780784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829222.780792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829222.780793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829222.780799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829222.780802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829222.780806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829222.780808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829222.780811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829222.780814:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829222.972723:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298043052, at 4298043072
00000100:00000040:3.0:1787829222.972724:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298043052, at 4298043072
00000100:00000010:2.0:1787829222.972741:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e631e0.
00000100:00000010:3.0:1787829222.972745:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63840.
00000020:00000001:2.0:1787829223.100836:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829223.100839:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829223.100855:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829223.804758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829223.804766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829223.804769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829223.804775:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829223.804779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829223.804782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829223.804785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829223.804789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829223.804791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829224.124906:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829224.124911:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829224.124915:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829224.828790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829224.828798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829224.828800:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829224.828805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829224.828809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829224.828813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829224.828815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829224.828818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829224.828821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1787829225.341006:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829225.341078:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829225.341091:0:2337:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829225.341096:0:2337:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829225.341102:0:2337:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829225.341108:0:2337:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829225.341114:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829225.341119:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:2.0:1787829225.341126:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829225.341132:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829225.341141:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98330
00000400:00000010:2.0:1787829225.341145:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98330.
00000100:00000001:2.0:1787829225.341151:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829225.341163:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829225.341202:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829225.341208:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829225.341218:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829225.341220:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829225.341234:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829225.341238:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955d00.
00000400:00000200:2.0:1787829225.341244:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829225.341248:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829225.341252:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98dd0
00000400:00000010:2.0:1787829225.341254:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98dd0.
00000100:00000001:2.0:1787829225.341258:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829225.341266:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341274:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829225.341278:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829225.341285:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787829225.341288:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341293:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829225.341302:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829225.341305:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:1.0:1787829225.341317:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829222/real 1787829225]  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1787829225.341318:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829225.341325:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:2.0:1787829225.341333:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829225.341347:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829225.341352:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98110
00000400:00000010:2.0:1787829225.341356:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98110.
00000100:00000001:2.0:1787829225.341363:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:1.0:1787829225.341368:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000200:2.0:1787829225.341370:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829225.341375:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829225.341380:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630d68.
00000100:00000001:2.0:1787829225.341386:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:1.0:1787829225.341391:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98a18
00000100:00000040:2.0:1787829225.341394:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787829225.341397:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98a18.
00000100:00000001:1.0:1787829225.341410:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787829225.341411:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829225.341416:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000200:1.0:1787829225.341422:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1787829225.341431:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829225.341439:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955900.
00000400:00000200:2.0:1787829225.341446:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829225.341456:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000200:1.0:1787829225.341457:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829225.341460:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98990
00000400:00000010:2.0:1787829225.341464:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98990.
00000100:00000001:2.0:1787829225.341470:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787829225.341479:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829225.341487:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:2.0:1787829225.341490:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341491:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.341496:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829225.341510:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829225.341515:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829225.341528:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829225.341538:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829225.341540:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829225.341541:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787829225.341556:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341565:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829225.341569:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341570:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.341572:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341574:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829225.341577:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829225.341581:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829225.341586:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829225.341589:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829225.341594:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829225.341603:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322624:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829225.341608:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829225.341613:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829222/real 1787829225]  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829225.341621:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829225.341623:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829225.341624:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829225.341628:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829225.341631:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829225.341634:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829225.341637:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829225.341646:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341653:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829225.341718:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341721:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.341722:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829225.341727:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341734:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829225.341738:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341744:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829225.341747:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341748:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.341750:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341752:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829225.341753:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829225.341755:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829225.341758:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829225.341760:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829225.341764:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829225.341771:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327322496:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829225.341775:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829225.341778:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829225.341781:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327322624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341787:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829225.341790:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829225.341794:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829225.341799:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829225.341802:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829225.341804:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829225.341807:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829225.341809:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:1.0:1787829225.341813:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829225.341815:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829225.341817:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:1.0:1787829225.341820:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829225.341822:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341824:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829225.341842:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327322496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341850:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829225.341852:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829225.341854:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829225.341858:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829225.341859:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829225.341861:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829225.341863:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829225.341864:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:1.0:1787829225.341867:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829225.341869:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829225.341870:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:1.0:1787829225.341873:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829225.341874:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341877:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829225.341880:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829225.341882:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829225.341885:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829225.341893:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829222/real 1787829225]  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829225.341903:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829225.341905:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829225.341908:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd946304d0.
00000400:00000200:1.0:1787829225.341911:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829225.341913:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829225.341917:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829225.341921:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829225.341929:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341936:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829225.341937:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341940:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.341941:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829225.341945:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341952:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829225.341958:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.341965:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829225.341968:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341970:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.341972:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.341975:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829225.341978:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829225.341982:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829225.341987:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829225.341990:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829225.341996:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829225.342006:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322752:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829225.342013:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829225.342019:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829222/real 1787829225]  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829225.342027:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829225.342028:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829225.342030:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829225.342046:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829225.342049:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829225.342051:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829225.342055:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829225.342063:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.342069:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829225.342071:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342074:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.342075:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829225.342079:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.342086:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829225.342090:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.342098:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829225.342101:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342103:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829225.342109:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342111:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829225.342114:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829225.342117:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829225.342121:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829225.342124:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829225.342128:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829225.342136:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327322880:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829225.342141:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342143:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829225.342146:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327322752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.342152:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829225.342153:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829225.342155:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829225.342160:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829225.342161:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829225.342163:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829225.342165:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829225.342167:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:1.0:1787829225.342170:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829225.342171:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829225.342172:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829225.342175:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829225.342176:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342178:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829225.342181:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327322880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829225.342188:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829225.342189:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829225.342191:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829225.342194:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829225.342195:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829225.342196:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829225.342198:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829225.342199:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:1.0:1787829225.342201:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829225.342202:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829225.342203:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:1.0:1787829225.342207:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829225.342208:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342210:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829225.342218:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829225.342220:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829225.342222:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829225.342223:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829225.852750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829225.852759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829225.852762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829225.852769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829225.852773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829225.852777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829225.852779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829225.852781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829225.852784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829226.172889:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829226.172895:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829226.172900:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829226.876791:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829226.876799:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829226.876802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829226.876807:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829226.876810:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829226.876814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829226.876830:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829226.876833:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829226.876835:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829227.196794:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829227.196798:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829227.196801:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829227.388790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829227.388804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829227.388808:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829227.388822:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829227.388828:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829227.388837:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3375
00000400:00000200:1.0:1787829227.388849:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829227.388857:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829227.388862:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829227.388866:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.388870:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.388873:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.388876:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829227.388880:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829227.388883:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829227.388886:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829227.388891:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829227.388903:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829227.388906:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829227.388909:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.388911:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.388913:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.388916:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829227.388920:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829227.388924:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829227.388928:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829227.388931:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829227.388936:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829227.388939:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.388942:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829227.388945:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829227.388959:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829227.388964:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829227.388969:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829227.388971:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.388975:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829227.388978:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829227.388984:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829227.388994:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829227.389004:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829227.389016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327323008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829227.389036:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829227.389042:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829227.389048:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327323008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829227.389075:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829227.389079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829227.389083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829227.389084:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829227.389086:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829227.389090:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829227.389092:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3375
00000400:00000200:1.0:1787829227.389096:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829227.389099:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829227.389102:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829227.389105:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389108:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.389110:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389112:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829227.389115:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829227.389116:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829227.389118:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829227.389121:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829227.389123:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829227.389125:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829227.389128:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389131:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.389134:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389136:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829227.389138:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829227.389141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389144:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829227.389146:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829227.389153:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829227.389156:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389158:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829227.389164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829227.389168:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389170:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829227.389175:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829227.389178:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829227.389180:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829227.389182:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389184:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829227.389186:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829227.389188:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829227.389192:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829227.389196:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829227.389202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327323136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829227.389210:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829227.389213:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829227.389217:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327323136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829227.389226:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829227.389229:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829227.389233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829227.389235:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829227.389238:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829227.389242:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829227.389250:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3375
00000400:00000200:1.0:1787829227.389256:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829227.389259:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829227.389263:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829227.389268:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389273:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.389277:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389282:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829227.389286:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829227.389288:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829227.389292:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829227.389296:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829227.389301:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829227.389307:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829227.389314:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389317:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.389319:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389322:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829227.389326:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829227.389329:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389335:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829227.389341:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829227.389345:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829227.389351:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389355:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829227.389360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829227.389365:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389368:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829227.389375:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829227.389380:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829227.389385:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829227.389387:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389393:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829227.389398:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829227.389400:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829227.389412:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829227.389418:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829227.389428:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327323264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829227.389440:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829227.389445:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829227.389452:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327323264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829227.389469:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829227.389475:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829227.389481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829227.389487:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829227.389497:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829227.389503:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829227.389510:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3375
00000400:00000200:1.0:1787829227.389519:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829227.389523:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829227.389525:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829227.389529:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389531:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.389533:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389536:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829227.389538:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829227.389540:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829227.389542:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829227.389545:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829227.389547:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829227.389550:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829227.389553:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389556:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829227.389558:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389560:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829227.389562:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829227.389565:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829227.389568:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829227.389570:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829227.389573:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829227.389575:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389577:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829227.389579:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829227.389582:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389584:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829227.389588:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829227.389591:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829227.389594:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829227.389596:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829227.389598:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829227.389600:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829227.389602:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829227.389607:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829227.389611:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829227.389617:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327323392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829227.389626:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829227.389628:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829227.389633:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327323392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829227.389642:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829227.389647:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3385)
00000800:00000200:0.1:1787829227.390022:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829227.390026:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829227.391077:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829227.391080:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829227.391546:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829227.391549:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829227.391924:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829227.391927:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787829227.900748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829227.900757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829227.900760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829227.900766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829227.900771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829227.900775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829227.900778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829227.900782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829227.900785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829228.092727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298048173, at 4298048192
00000100:00000040:3.0:1787829228.092727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298048173, at 4298048192
00000100:00000010:2.0:1787829228.092746:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000010:3.0:1787829228.092748:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000100:00000001:2.0:1787829228.924764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829228.924770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829228.924772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829228.924776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829228.924779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829228.924781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829228.924783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829228.924785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829228.924787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829229.244880:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829229.244885:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829229.244888:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829229.948721:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829229.948730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829229.948733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829229.948738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829229.948742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829229.948745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829229.948748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829229.948751:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829229.948754:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829230.268872:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829230.268876:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829230.268880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829230.972768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829230.972777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829230.972780:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829230.972785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829230.972806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829230.972810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829230.972814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829230.972817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829230.972820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829231.996751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829231.996758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829231.996760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829231.996766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829231.996770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829231.996774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829231.996776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829231.996778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829231.996790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829232.316898:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829232.316902:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829232.316906:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829232.508792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829232.508804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829232.508807:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829232.508811:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829232.508816:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829232.508824:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3380
00000400:00000200:1.0:1787829232.508834:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829232.508840:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829232.508844:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829232.508847:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.508850:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.508853:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.508856:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829232.508861:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829232.508863:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829232.508865:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829232.508868:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829232.508870:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829232.508872:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829232.508874:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.508876:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.508877:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.508879:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829232.508881:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829232.508884:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829232.508886:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829232.508889:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829232.508893:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829232.508895:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.508897:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829232.508899:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829232.508910:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829232.508912:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829232.508916:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829232.508918:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.508921:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829232.508924:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829232.508928:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829232.508936:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829232.508943:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829232.508953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327323520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829232.508961:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829232.508965:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829232.508969:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327323520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829232.508988:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829232.508991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829232.508994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829232.508995:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829232.508996:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829232.508998:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829232.509000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3380
00000400:00000200:1.0:1787829232.509002:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829232.509004:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829232.509006:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829232.509008:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509011:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.509014:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509016:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829232.509019:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829232.509021:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829232.509023:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829232.509026:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829232.509030:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829232.509032:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829232.509036:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509039:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.509042:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509044:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829232.509047:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829232.509052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509059:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829232.509063:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829232.509072:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829232.509075:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509077:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829232.509083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829232.509089:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509094:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829232.509104:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829232.509111:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829232.509115:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829232.509118:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509128:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829232.509130:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829232.509133:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829232.509138:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829232.509144:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829232.509155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327323648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829232.509174:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829232.509175:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829232.509179:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327323648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829232.509189:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829232.509193:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829232.509200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829232.509202:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829232.509203:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829232.509206:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829232.509211:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3380
00000400:00000200:1.0:1787829232.509219:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829232.509223:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829232.509226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829232.509228:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509231:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.509234:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509239:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829232.509248:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829232.509250:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829232.509251:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829232.509253:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829232.509256:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829232.509259:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829232.509269:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509271:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.509274:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509283:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829232.509286:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829232.509291:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509295:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829232.509299:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829232.509302:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829232.509304:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509308:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829232.509311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829232.509324:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509330:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829232.509334:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829232.509338:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829232.509343:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829232.509345:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509354:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829232.509357:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829232.509361:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829232.509367:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829232.509372:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829232.509383:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327323776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829232.509399:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829232.509401:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829232.509407:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327323776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829232.509420:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829232.509423:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829232.509430:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829232.509433:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829232.509435:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829232.509445:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829232.509450:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3380
00000400:00000200:1.0:1787829232.509501:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829232.509507:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829232.509512:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829232.509521:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509525:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.509532:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509535:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829232.509548:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829232.509551:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829232.509555:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829232.509560:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829232.509563:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829232.509564:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829232.509569:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509573:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829232.509576:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509579:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829232.509580:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829232.509583:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829232.509584:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829232.509585:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829232.509588:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829232.509592:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509595:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829232.509598:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829232.509606:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509608:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829232.509611:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829232.509617:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829232.509622:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829232.509624:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829232.509630:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.1:1787829232.509632:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:1.0:1787829232.509633:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829232.509635:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829232.509642:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000800:00000200:0.1:1787829232.509642:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1787829232.509649:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829232.509681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327323904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829232.509688:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829232.509690:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829232.509693:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327323904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829232.509699:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829232.509709:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3390)
00000800:00000200:0.1:1787829232.510207:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829232.510210:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829232.510534:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829232.510537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829232.510903:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829232.510906:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829233.020849:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829233.020863:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829233.020865:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829233.020870:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829233.020874:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829233.020877:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829233.020880:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829233.020883:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829233.020885:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829233.212768:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298053293, at 4298053312
00000100:00000040:2.0:1787829233.212771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298053293, at 4298053312
00000100:00000010:3.0:1787829233.212800:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000010:2.0:1787829233.212802:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000020:00000001:2.0:1787829233.340872:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829233.340878:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829233.340882:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829234.044740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829234.044746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829234.044747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829234.044751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829234.044754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829234.044757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829234.044759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829234.044761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829234.044763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829235.068752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829235.068770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829235.068773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829235.068779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829235.068783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829235.068787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829235.068789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829235.068792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829235.068795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829235.388867:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829235.388870:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829235.388873:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829236.092774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829236.092780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829236.092782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829236.092785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829236.092789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829236.092791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829236.092794:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829236.092796:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829236.092798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829236.412888:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829236.412891:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829236.412894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829237.116775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829237.116781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829237.116784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829237.116789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829237.116793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829237.116796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829237.116799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829237.116802:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829237.116805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829237.628799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829237.628809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829237.628812:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829237.628817:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829237.628821:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829237.628829:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3385
00000400:00000200:1.0:1787829237.628838:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829237.628845:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829237.628849:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829237.628853:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.628856:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.628859:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.628862:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829237.628867:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829237.628869:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829237.628872:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829237.628875:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829237.628878:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829237.628882:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829237.628885:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.628888:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.628890:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.628892:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829237.628895:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829237.628899:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829237.628902:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829237.628905:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829237.628909:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829237.628913:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.628916:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829237.628919:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829237.628929:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829237.628933:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829237.628937:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829237.628939:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.628942:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829237.628946:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829237.628950:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829237.628958:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829237.628965:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829237.628975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327324032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829237.628987:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829237.628993:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829237.628998:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327324032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829237.629019:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829237.629023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829237.629027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829237.629029:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829237.629031:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829237.629035:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829237.629038:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3385
00000400:00000200:1.0:1787829237.629044:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829237.629048:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829237.629062:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829237.629071:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629077:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.629082:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629088:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829237.629093:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829237.629097:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829237.629102:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829237.629105:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829237.629110:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829237.629115:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829237.629122:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629128:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.629133:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629137:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829237.629143:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829237.629150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629155:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829237.629157:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829237.629165:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829237.629169:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629173:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829237.629180:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829237.629186:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629189:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829237.629195:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829237.629200:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829237.629208:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829237.629213:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629218:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829237.629221:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829237.629223:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829237.629229:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829237.629236:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829237.629248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327324160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829237.629263:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829237.629268:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829237.629277:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327324160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829237.629291:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829237.629296:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829237.629303:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829237.629309:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829237.629313:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829237.629321:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829237.629328:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3385
00000400:00000200:1.0:1787829237.629357:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829237.629367:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829237.629371:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829237.629375:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629378:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.629384:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629389:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829237.629394:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829237.629397:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829237.629401:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829237.629404:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829237.629408:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829237.629415:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829237.629426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829237.629429:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829237.629431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:1.0:1787829237.629433:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.629439:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629445:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829237.629451:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829237.629457:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629461:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829237.629467:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829237.629472:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829237.629480:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629484:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829237.629489:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829237.629495:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629500:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829237.629507:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829237.629513:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829237.629518:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829237.629522:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629526:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829237.629530:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829237.629533:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829237.629540:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829237.629553:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829237.629564:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327324288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829237.629578:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829237.629582:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829237.629588:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327324288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829237.629606:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829237.629612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829237.629616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829237.629618:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829237.629619:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829237.629623:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829237.629628:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3385
00000400:00000200:1.0:1787829237.629636:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829237.629642:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829237.629648:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829237.629692:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629697:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.629703:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629709:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829237.629713:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829237.629715:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829237.629717:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829237.629722:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829237.629726:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829237.629730:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829237.629738:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629754:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829237.629759:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629765:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829237.629772:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829237.629779:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829237.629785:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829237.629790:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829237.629795:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829237.629801:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629807:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829237.629812:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829237.629816:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629820:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829237.629826:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829237.629831:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829237.629836:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829237.629839:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829237.629842:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829237.629845:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829237.629849:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829237.629855:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829237.629863:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829237.629876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327324416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829237.629895:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829237.629899:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829237.629909:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327324416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829237.629929:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829237.629937:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3395)
00000800:00000200:0.1:1787829237.630146:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829237.630148:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829237.630448:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829237.630451:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829237.630792:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829237.630795:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829238.140763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829238.140769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829238.140771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829238.140775:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829238.140779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829238.140782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829238.140784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829238.140786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829238.140789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829238.332714:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298058412, at 4298058432
00000100:00000040:2.0:1787829238.332720:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298058412, at 4298058432
00000100:00000010:2.0:1787829238.332749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000100:00000010:3.0:1787829238.332752:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000020:00000001:2.0:1787829238.460843:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829238.460846:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829238.460848:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829239.164749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829239.164758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829239.164761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829239.164767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829239.164771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829239.164775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829239.164778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829239.164782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829239.164786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829239.484882:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829239.484887:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829239.484891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829240.188703:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829240.188711:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829240.188713:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829240.188718:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829240.188723:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829240.188737:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829240.188741:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829240.188744:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829240.188748:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829241.212771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829241.212781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829241.212783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829241.212788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829241.212791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829241.212795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829241.212797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829241.212800:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829241.212803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829241.532970:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829241.532974:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829241.532977:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829242.236754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829242.236760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829242.236761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829242.236765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829242.236768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829242.236771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829242.236773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829242.236775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829242.236777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829242.556871:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829242.556875:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829242.556878:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829242.748780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829242.748796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829242.748800:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829242.748805:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829242.748812:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829242.748822:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3390
00000400:00000200:1.0:1787829242.748835:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829242.748844:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829242.748849:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829242.748854:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.748859:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.748862:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.748866:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829242.748872:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829242.748876:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829242.748879:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829242.748883:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829242.748886:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829242.748890:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829242.748894:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.748896:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.748898:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.748901:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829242.748906:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829242.748910:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829242.748914:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829242.748918:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829242.748923:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829242.748927:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.748930:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829242.748934:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829242.748946:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829242.748950:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829242.748955:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829242.748958:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.748963:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829242.748967:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829242.748972:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829242.748983:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829242.749001:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829242.749015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327324544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829242.749029:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829242.749050:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829242.749056:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327324544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829242.749085:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829242.749089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829242.749093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829242.749095:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829242.749096:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829242.749100:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829242.749103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3390
00000400:00000200:1.0:1787829242.749108:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829242.749111:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829242.749114:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829242.749117:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749119:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.749122:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749124:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829242.749127:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829242.749128:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829242.749131:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829242.749133:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829242.749135:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829242.749138:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829242.749142:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749145:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.749148:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749152:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829242.749155:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829242.749158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749162:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829242.749166:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829242.749174:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829242.749179:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749185:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829242.749191:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829242.749198:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749204:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829242.749212:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829242.749219:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829242.749226:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829242.749230:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749236:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829242.749240:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829242.749244:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829242.749251:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829242.749258:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829242.749268:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327324672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829242.749282:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829242.749287:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829242.749295:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327324672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829242.749310:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829242.749316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829242.749324:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829242.749328:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829242.749331:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829242.749349:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829242.749356:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3390
00000400:00000200:1.0:1787829242.749364:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829242.749369:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829242.749372:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829242.749380:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749386:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.749393:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749398:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829242.749404:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829242.749407:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829242.749411:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829242.749415:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829242.749421:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829242.749427:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829242.749435:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749438:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.749441:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749462:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829242.749467:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829242.749475:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749481:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829242.749485:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829242.749491:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829242.749495:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749499:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829242.749504:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829242.749511:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829242.749512:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829242.749516:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829242.749517:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000010:1.0:1787829242.749525:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829242.749532:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829242.749536:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829242.749537:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749541:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829242.749543:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829242.749546:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829242.749555:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829242.749563:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829242.749575:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327324800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829242.749596:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829242.749601:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829242.749609:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327324800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829242.749638:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829242.749642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829242.749650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829242.749653:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829242.749657:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829242.749660:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829242.749665:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3390
00000400:00000200:1.0:1787829242.749698:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829242.749705:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829242.749741:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829242.749747:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749750:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.749756:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749761:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829242.749767:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829242.749769:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829242.749774:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829242.749778:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829242.749782:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829242.749803:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829242.749811:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749817:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829242.749822:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749827:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829242.749831:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829242.749838:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829242.749846:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829242.749850:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829242.749854:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829242.749860:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749865:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829242.749870:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829242.749877:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749882:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829242.749889:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829242.749896:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829242.749901:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829242.749905:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829242.749910:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829242.749913:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829242.749917:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829242.749925:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829242.749932:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829242.749945:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327324928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829242.749966:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829242.749971:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829242.749981:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327324928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829242.750019:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829242.750027:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3400)
00000800:00000200:0.1:1787829242.750166:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829242.750168:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829242.750429:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829242.750431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829242.750745:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829242.750747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787829243.260790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829243.260800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829243.260803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829243.260811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829243.260816:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829243.260821:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829243.260825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829243.260828:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829243.260832:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829243.452732:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298063533, at 4298063552
00000100:00000040:3.0:1787829243.452747:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298063533, at 4298063552
00000100:00000010:2.0:1787829243.452761:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000100:00000010:3.0:1787829243.452764:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000100:00000001:2.0:1787829244.284819:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829244.284829:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829244.284832:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829244.284839:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829244.284844:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829244.284849:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829244.284853:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829244.284857:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829244.284860:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829244.604878:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829244.604884:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829244.604889:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829245.628997:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829245.629004:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829245.629009:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829247.676985:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829247.676994:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829247.677002:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829247.868820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829247.868836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829247.868840:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829247.868846:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829247.868854:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829247.868865:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3395
00000400:00000200:1.0:1787829247.868878:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829247.868888:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829247.868894:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829247.868899:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.868905:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.868909:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.868913:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829247.868919:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829247.868922:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829247.868927:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829247.868932:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829247.868936:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829247.868940:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829247.868945:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.868949:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.868952:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.868955:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829247.868960:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829247.868966:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829247.868970:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829247.868975:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829247.868980:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829247.868984:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.868988:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829247.868992:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829247.869007:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829247.869012:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829247.869018:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829247.869021:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.869027:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829247.869031:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829247.869037:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829247.869067:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829247.869077:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829247.869090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327325056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829247.869104:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829247.869110:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829247.869117:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327325056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829247.869147:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829247.869151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829247.869156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829247.869158:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829247.869161:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829247.869165:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829247.869168:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3395
00000400:00000200:1.0:1787829247.869173:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829247.869178:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829247.869181:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829247.869186:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869190:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.869195:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869200:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829247.869219:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829247.869222:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829247.869227:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829247.869232:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829247.869239:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829247.869248:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829247.869261:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869272:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.869284:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869292:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829247.869304:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829247.869336:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869345:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829247.869361:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829247.869373:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829247.869380:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.869387:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829247.869395:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829247.869401:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.869409:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829247.869422:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829247.869432:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829247.869442:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829247.869449:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.869457:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829247.869460:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829247.869463:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829247.869475:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829247.869485:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829247.869525:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327325184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829247.869556:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829247.869563:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829247.869588:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327325184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829247.869652:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829247.869658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829247.869664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829247.869667:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829247.869728:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829247.869738:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829247.869748:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3395
00000800:00000200:0.1:1787829247.869757:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:1.0:1787829247.869764:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.1:1787829247.869766:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000400:00000200:1.0:1787829247.869778:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829247.869791:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829247.869804:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869818:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.869831:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869846:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829247.869857:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829247.869864:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829247.869872:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829247.869879:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829247.869891:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829247.869909:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829247.869922:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869929:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.869936:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869942:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829247.869950:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829247.869960:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829247.869966:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829247.869969:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829247.869976:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829247.869987:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.869999:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829247.870006:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829247.870023:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.870032:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829247.870046:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829247.870058:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829247.870086:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829247.870090:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.870096:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829247.870100:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829247.870104:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829247.870116:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829247.870124:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829247.870143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327325312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829247.870176:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829247.870184:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829247.870199:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327325312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829247.870252:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829247.870264:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829247.870277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829247.870287:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829247.870293:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829247.870305:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829247.870316:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3395
00000400:00000200:1.0:1787829247.870334:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829247.870349:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829247.870357:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829247.870368:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.870374:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.870386:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.870398:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829247.870405:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829247.870411:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829247.870422:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829247.870435:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829247.870447:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829247.870460:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829247.870474:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.870479:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829247.870485:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829247.870516:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829247.870534:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829247.870549:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829247.870561:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829247.870571:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829247.870579:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829247.870589:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.870600:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829247.870611:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829247.870622:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.870632:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829247.870646:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829247.870655:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829247.870663:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829247.870668:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829247.870746:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829247.870756:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829247.870764:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829247.870782:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000010:1.0:1787829247.870784:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000800:00000200:0.1:1787829247.870785:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1787829247.870801:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829247.870828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327325440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829247.870873:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829247.870881:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829247.870898:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327325440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829247.870945:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829247.870960:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3405)
00000800:00000200:0.1:1787829247.871166:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829247.871168:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829247.871405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829247.871406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1787829248.572765:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298068653, at 4298068672
00000100:00000040:3.0:1787829248.572765:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298068653, at 4298068672
00000100:00000010:2.0:1787829248.572779:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000100:00000010:3.0:1787829248.572780:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000020:00000001:2.0:1787829248.700936:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829248.700941:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829248.700944:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829250.748856:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829250.748861:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829250.748865:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829251.772833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829251.772838:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829251.772843:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829252.988833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829252.988845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829252.988849:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829252.988854:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829252.988859:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829252.988868:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3400
00000400:00000200:1.0:1787829252.988878:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829252.988885:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829252.988889:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829252.988892:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.988896:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.988907:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.988910:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829252.988915:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829252.988917:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829252.988920:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829252.988925:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829252.988927:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829252.988930:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829252.988933:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.988936:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.988937:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.988940:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829252.988943:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829252.988946:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829252.988949:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829252.988952:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829252.988956:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829252.988960:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.988963:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829252.988966:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829252.988978:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829252.988982:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829252.988986:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829252.988988:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.988992:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829252.988995:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829252.989001:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829252.989010:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829252.989018:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829252.989029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327325568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829252.989041:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829252.989045:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829252.989050:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327325568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829252.989072:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829252.989075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829252.989079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829252.989081:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829252.989083:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829252.989086:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829252.989089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3400
00000400:00000200:1.0:1787829252.989093:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829252.989097:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829252.989100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829252.989104:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989108:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.989112:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989117:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829252.989121:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829252.989123:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829252.989125:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829252.989129:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829252.989134:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829252.989142:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829252.989150:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989152:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.989165:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989170:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829252.989174:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829252.989179:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989186:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829252.989190:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829252.989196:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829252.989200:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989206:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829252.989213:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829252.989218:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989223:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829252.989229:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829252.989234:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829252.989243:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829252.989246:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989252:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829252.989256:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829252.989261:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829252.989270:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829252.989279:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829252.989290:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327325696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829252.989302:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829252.989306:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829252.989316:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327325696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829252.989329:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829252.989333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829252.989340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829252.989343:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829252.989345:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829252.989348:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829252.989353:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3400
00000400:00000200:1.0:1787829252.989360:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829252.989362:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829252.989365:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829252.989371:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989375:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.989379:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989384:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829252.989388:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829252.989391:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829252.989392:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829252.989396:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829252.989399:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829252.989403:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829252.989409:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989413:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.989418:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829252.989420:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829252.989421:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829252.989425:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000020:00000040:1.0:1787829252.989432:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829252.989438:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989441:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829252.989443:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829252.989447:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829252.989453:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989455:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829252.989458:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829252.989462:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989465:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829252.989469:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829252.989474:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829252.989479:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829252.989481:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989485:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829252.989497:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829252.989501:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829252.989508:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829252.989514:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829252.989524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327325824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829252.989535:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829252.989538:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829252.989546:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327325824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829252.989557:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829252.989562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829252.989565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829252.989567:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829252.989571:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829252.989573:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829252.989575:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3400
00000400:00000200:1.0:1787829252.989579:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829252.989586:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829252.989593:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829252.989601:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989608:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.989614:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989621:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829252.989627:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829252.989630:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829252.989647:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829252.989650:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829252.989678:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829252.989682:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829252.989687:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989690:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829252.989697:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989703:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829252.989708:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829252.989717:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829252.989724:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829252.989730:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829252.989734:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829252.989741:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989745:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829252.989750:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829252.989754:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989758:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829252.989764:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829252.989768:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829252.989774:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829252.989777:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829252.989780:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829252.989782:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829252.989785:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829252.989790:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829252.989796:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829252.989806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327325952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829252.989823:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829252.989826:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829252.989834:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327325952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829252.989850:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829252.989856:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3410)
00000800:00000200:0.1:1787829252.990005:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829252.990006:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829252.990199:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829252.990201:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829252.990377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829252.990379:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000040:2.0:1787829253.692737:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298073772, at 4298073792
00000100:00000040:3.0:1787829253.692737:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298073772, at 4298073792
00000100:00000010:2.0:1787829253.692752:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000100:00000010:3.0:1787829253.692754:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000020:00000001:2.0:1787829253.820841:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829253.820844:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829253.820847:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829254.844853:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829254.844858:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829254.844863:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829256.892873:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829256.892877:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829256.892880:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829257.916844:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829257.916848:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829257.916851:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829258.109033:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829258.109040:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829258.109045:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829258.109056:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109074:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829258.109079:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.109087:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829258.109093:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829258.109101:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829258.109109:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327326080:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829258.109118:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829258.109124:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829258.109130:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.109136:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829258.109145:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829258.109150:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829258.109159:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829258.109167:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829258.109178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829258.109188:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326080, portal 25
00000100:00000001:1.0:1787829258.109194:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829258.109200:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829258.109213:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109236:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829258.109244:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829258.109250:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829258.109258:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327326080, offset 0
00000400:00000200:1.0:1787829258.109274:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829258.109288:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.109300:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829258.109311:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829258.109319:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829258.109326:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829258.109334:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829258.109344:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40319:40319:256:4294967295] 192.168.203.116@tcp LPNI seq info [40319:40319:8:4294967295]
00000400:00000200:1.0:1787829258.109356:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.109378:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829258.109394:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.109403:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829258.109418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829258.109428:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787829258.109440:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829258.109448:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:1.0:1787829258.109461:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.109467:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829258.109477:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829258.109488:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829258.109498:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829258.109503:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829258.109510:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829258.109522:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109543:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.109550:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109561:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.109563:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829258.109571:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829258.109576:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829258.109580:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298078892
00000100:00000010:1.0:1787829258.109587:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63240.
00000100:00000001:1.0:1787829258.109594:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829258.109615:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298078892
00000100:00000010:1.0:1787829258.109620:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63060.
00000100:00000001:1.0:1787829258.109624:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829258.109634:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829258.109638:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.109641:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829258.109646:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829258/real 1787829258]  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829258.109681:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829258.109684:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829258.109687:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829258.109692:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829258.109694:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829258.109697:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829258.109702:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829258.109713:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109722:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829258.109724:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.109727:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.109729:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829258.109734:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109743:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829258.109749:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109758:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829258.109761:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.109763:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.109764:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.109767:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829258.109770:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829258.109773:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829258.109776:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829258.109777:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829258.109781:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829258.109791:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327326080:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829258.109798:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.109800:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.109804:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327326080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109812:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829258.109815:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829258.109819:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829258.109822:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829258.109824:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829258.109826:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829258.109829:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829258.109832:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829258.109836:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829258.109838:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829258.109840:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829258.109843:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829258.109844:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.109846:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.109854:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829258.109856:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.109859:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829258.109861:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829258.109862:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829258.109867:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109885:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829258.109887:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.109889:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829258.109891:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829258.109894:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829258.109897:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326208:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829258.109901:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829258.109903:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829258.109905:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.109908:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829258.109911:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829258.109913:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829258.109918:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829258.109921:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829258.109925:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829258.109928:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326208, portal 10
00000100:00000001:1.0:1787829258.109930:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829258.109932:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829258.109940:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.109947:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829258.109950:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829258.109953:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829258.109955:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327326208, offset 0
00000400:00000200:1.0:1787829258.109960:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829258.109964:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.109969:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829258.109972:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829258.109975:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829258.109978:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829258.109981:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829258.109984:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40320:40320:256:4294967295] 192.168.203.116@tcp LPNI seq info [40320:40320:8:4294967295]
00000400:00000200:1.0:1787829258.109989:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.109996:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829258.110001:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.110004:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829258.110009:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829258.110012:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787829258.110017:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829258.110021:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:1.0:1787829258.110025:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.110028:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829258.110031:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829258.110035:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829258.110039:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829258.110041:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829258.110043:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829258.110047:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110056:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.110059:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110067:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.110069:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829258.110072:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829258.110085:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829258.110089:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829258.110091:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298078219
00000100:00000001:1.0:1787829258.110094:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298078892
00000100:00000001:1.0:1787829258.110097:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298078219
00000100:00000001:1.0:1787829258.110099:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298078892
00000100:00000001:1.0:1787829258.110101:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829258.110103:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.110106:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829258.110110:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829258/real 1787829258]  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829258.110118:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829258.110120:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829258.110122:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829258.110125:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829258.110127:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829258.110130:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829258.110134:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829258.110144:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110151:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829258.110153:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110155:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.110157:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829258.110161:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110168:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829258.110172:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110180:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829258.110183:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110184:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.110186:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110188:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829258.110190:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829258.110192:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829258.110196:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829258.110198:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829258.110202:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829258.110211:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326208:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829258.110215:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829258.110219:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110233:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829258.110235:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.110238:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829258.110239:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829258.110242:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829258.110245:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829258.110248:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829258.110250:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829258.110252:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.110254:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829258.110257:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829258.110260:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829258.110263:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829258.110266:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829258.110270:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829258.110273:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326336, portal 4
00000100:00000001:1.0:1787829258.110275:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829258.110277:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829258.110283:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110290:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829258.110293:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829258.110295:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829258.110298:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327326336, offset 0
00000400:00000200:1.0:1787829258.110302:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829258.110305:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.110309:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829258.110312:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829258.110315:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829258.110317:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829258.110320:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829258.110324:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40321:40321:256:4294967295] 192.168.203.116@tcp LPNI seq info [40321:40321:8:4294967295]
00000400:00000200:1.0:1787829258.110329:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.110337:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829258.110342:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.110345:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829258.110350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829258.110353:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787829258.110357:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829258.110360:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:1.0:1787829258.110365:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.110367:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829258.110370:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829258.110374:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829258.110377:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829258.110379:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829258.110382:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829258.110385:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110394:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.110398:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110406:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.110407:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829258.110410:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829258.110413:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829258.110417:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829258.110419:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298078226
00000100:00000001:1.0:1787829258.110422:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298078892
00000100:00000001:1.0:1787829258.110425:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298078226
00000100:00000001:1.0:1787829258.110427:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298078892
00000100:00000001:1.0:1787829258.110429:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829258.110432:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.110434:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829258.110438:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829258/real 1787829258]  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829258.110447:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829258.110449:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829258.110450:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829258.110453:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829258.110455:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829258.110457:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829258.110460:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829258.110470:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110477:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829258.110479:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110481:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.110482:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829258.110486:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110493:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829258.110497:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110504:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829258.110507:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110508:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.110510:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110511:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829258.110513:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829258.110516:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829258.110520:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829258.110522:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829258.110527:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829258.110535:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829258.110539:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829258.110543:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110549:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829258.110551:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.110553:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829258.110554:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829258.110557:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829258.110560:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326464:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829258.110564:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829258.110565:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829258.110567:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829258.110569:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829258.110573:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829258.110575:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829258.110578:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829258.110581:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829258.110584:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829258.110587:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326464, portal 4
00000100:00000001:1.0:1787829258.110589:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829258.110591:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829258.110597:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110619:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829258.110623:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829258.110626:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829258.110628:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327326464, offset 0
00000400:00000200:1.0:1787829258.110633:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829258.110637:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.110650:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829258.110653:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829258.110673:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829258.110676:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829258.110679:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829258.110683:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40322:40322:256:4294967295] 192.168.203.116@tcp LPNI seq info [40322:40322:8:4294967295]
00000400:00000200:1.0:1787829258.110688:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829258.110696:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829258.110701:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.110704:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829258.110709:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829258.110712:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787829258.110717:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829258.110720:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:1.0:1787829258.110724:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829258.110727:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829258.110730:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829258.110743:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829258.110746:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829258.110748:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829258.110751:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829258.110755:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110764:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.110768:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110776:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.110777:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829258.110781:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829258.110784:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829258.110788:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829258.110790:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298078232
00000100:00000001:1.0:1787829258.110793:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298078892
00000100:00000001:1.0:1787829258.110795:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298078232
00000100:00000001:1.0:1787829258.110798:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298078892
00000100:00000001:1.0:1787829258.110800:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829258.110803:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.110805:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829258.110808:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829258/real 1787829258]  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829258.110817:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829258.110819:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829258.110821:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829258.110823:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829258.110825:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829258.110827:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829258.110831:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829258.110841:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110858:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829258.110860:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110862:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.110864:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829258.110868:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110876:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829258.110880:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110889:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829258.110892:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110893:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829258.110895:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110897:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829258.110898:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829258.110901:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829258.110904:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829258.110906:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829258.110910:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829258.110919:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326464:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829258.110923:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110933:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.110936:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327326208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110943:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829258.110945:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829258.110947:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829258.110950:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829258.110952:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829258.110953:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829258.110956:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829258.110958:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829258.110960:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829258.110962:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829258.110964:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829258.110966:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829258.110968:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.110970:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.110973:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327326336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.110981:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829258.110983:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829258.110984:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829258.110986:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829258.110988:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829258.110989:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829258.110991:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829258.110993:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829258.110995:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829258.110996:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829258.110998:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829258.111000:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829258.111002:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.111004:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829258.111007:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327326464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829258.111014:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829258.111015:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829258.111017:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829258.111019:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829258.111020:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829258.111021:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829258.111023:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829258.111024:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829258.111026:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829258.111027:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829258.111029:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829258.111031:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829258.111032:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.111034:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829258.111037:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829258.111038:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829258.111040:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829258.111043:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829258.812788:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298078892, at 4298078912
00000100:00000040:3.0:1787829258.812802:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298078892, at 4298078912
00000100:00000010:2.0:1787829258.812822:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63240.
00000100:00000010:3.0:1787829258.812838:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63060.
00000020:00000001:2.0:1787829260.988928:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829260.988932:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829260.988936:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829263.229059:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829263.229072:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829263.229083:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829263.229106:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.229137:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829263.229142:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.229154:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829263.229161:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829263.229177:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829263.229203:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327326592:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829263.229224:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829263.229238:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829263.229242:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.229245:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829263.229256:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829263.229268:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829263.229292:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829263.229302:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829263.229315:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829263.229327:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326592, portal 25
00000100:00000001:1.0:1787829263.229334:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829263.229340:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829263.229353:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.229377:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829263.229383:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829263.229395:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829263.229403:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327326592, offset 0
00000400:00000200:1.0:1787829263.229421:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829263.229453:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.229470:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829263.229483:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829263.229507:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829263.229524:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829263.229531:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829263.229537:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40323:40323:256:4294967295] 192.168.203.116@tcp LPNI seq info [40323:40323:8:4294967295]
00000400:00000200:1.0:1787829263.229543:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.229583:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829263.229596:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.229610:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829263.229637:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829263.229704:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829263.229722:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829263.229747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829263.229763:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.229769:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829263.229777:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829263.229791:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829263.229799:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829263.229805:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829263.229815:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829263.229824:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.229856:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.229875:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.229913:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.229918:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829263.229931:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829263.229945:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829263.229961:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298084013
00000100:00000010:1.0:1787829263.229993:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63ae0.
00000100:00000001:1.0:1787829263.230005:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829263.230019:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298084013
00000100:00000010:1.0:1787829263.230028:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63f60.
00000100:00000001:1.0:1787829263.230036:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829263.230046:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829263.230057:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.230066:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829263.230083:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829263/real 1787829263]  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829263.230123:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829263.230134:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829263.230143:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829263.230154:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829263.230161:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829263.230166:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829263.230179:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829263.230225:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230247:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829263.230250:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230255:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.230261:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829263.230279:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230295:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829263.230311:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230326:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829263.230335:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230340:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.230347:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230355:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829263.230359:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829263.230371:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829263.230391:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829263.230401:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829263.230414:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829263.230439:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327326592:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829263.230451:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230458:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.230467:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327326592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230514:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829263.230519:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829263.230524:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829263.230532:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829263.230536:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829263.230539:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829263.230545:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829263.230552:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829263.230580:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829263.230583:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829263.230589:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829263.230594:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829263.230598:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230605:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230619:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829263.230623:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.230635:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829263.230643:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829263.230648:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829263.230677:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230691:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829263.230693:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.230701:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829263.230704:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829263.230707:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829263.230713:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326720:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829263.230720:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829263.230724:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829263.230726:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.230729:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829263.230733:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829263.230735:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829263.230740:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829263.230743:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829263.230748:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829263.230752:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326720, portal 10
00000100:00000001:1.0:1787829263.230755:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829263.230756:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829263.230762:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230767:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829263.230769:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829263.230771:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829263.230773:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327326720, offset 0
00000400:00000200:1.0:1787829263.230776:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829263.230779:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.230782:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829263.230785:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829263.230787:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829263.230789:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829263.230790:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829263.230794:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40324:40324:256:4294967295] 192.168.203.116@tcp LPNI seq info [40324:40324:8:4294967295]
00000400:00000200:1.0:1787829263.230797:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.230803:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829263.230806:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.230809:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829263.230812:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829263.230814:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829263.230818:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829263.230820:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829263.230823:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.230825:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829263.230828:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829263.230831:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829263.230834:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829263.230836:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829263.230839:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829263.230842:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230852:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.230856:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230863:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.230865:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829263.230869:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829263.230874:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829263.230878:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829263.230881:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298083340
00000100:00000001:1.0:1787829263.230884:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298084013
00000100:00000001:1.0:1787829263.230886:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298083340
00000100:00000001:1.0:1787829263.230888:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298084013
00000100:00000001:1.0:1787829263.230890:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829263.230893:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.230894:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829263.230897:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829263/real 1787829263]  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829263.230904:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829263.230905:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829263.230906:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829263.230909:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829263.230910:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829263.230912:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829263.230914:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829263.230920:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230924:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829263.230925:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230926:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.230927:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829263.230929:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230934:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829263.230936:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230942:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829263.230944:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230945:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.230945:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.230946:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829263.230948:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829263.230950:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829263.230953:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829263.230955:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829263.230958:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829263.230963:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326720:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829263.230966:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829263.230968:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.230973:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829263.230974:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.230975:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829263.230976:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829263.230978:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829263.230979:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829263.230981:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829263.230983:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829263.230984:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.230985:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829263.230988:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829263.230989:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829263.230992:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829263.230994:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829263.230996:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829263.230999:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326848, portal 4
00000100:00000001:1.0:1787829263.231000:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829263.231001:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829263.231006:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231011:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829263.231013:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829263.231015:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829263.231017:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327326848, offset 0
00000400:00000200:1.0:1787829263.231019:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829263.231022:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.231025:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829263.231027:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829263.231029:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829263.231031:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829263.231033:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829263.231035:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40325:40325:256:4294967295] 192.168.203.116@tcp LPNI seq info [40325:40325:8:4294967295]
00000400:00000200:1.0:1787829263.231039:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.231043:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829263.231046:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.231049:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829263.231052:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829263.231054:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829263.231057:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829263.231058:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829263.231061:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.231063:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829263.231065:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829263.231068:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829263.231070:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829263.231071:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829263.231074:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829263.231077:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231086:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.231090:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231098:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.231100:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829263.231104:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829263.231108:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829263.231112:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829263.231114:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298083347
00000100:00000001:1.0:1787829263.231117:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298084013
00000100:00000001:1.0:1787829263.231120:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298083347
00000100:00000001:1.0:1787829263.231122:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298084013
00000100:00000001:1.0:1787829263.231125:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829263.231127:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.231129:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829263.231132:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829263/real 1787829263]  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829263.231138:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829263.231139:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829263.231141:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829263.231143:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829263.231144:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829263.231146:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829263.231148:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829263.231154:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231159:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829263.231159:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231161:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.231161:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829263.231164:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231168:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829263.231170:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231176:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829263.231177:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231179:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.231180:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231181:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829263.231182:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829263.231183:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829263.231185:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829263.231187:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829263.231189:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829263.231194:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829263.231196:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231197:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.231199:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327326720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231203:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829263.231204:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829263.231206:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829263.231208:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829263.231209:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829263.231210:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829263.231212:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829263.231213:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829263.231216:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829263.231217:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829263.231218:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829263.231220:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829263.231221:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231223:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.231225:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327326848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231230:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829263.231231:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829263.231232:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829263.231233:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829263.231234:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829263.231234:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829263.231235:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829263.231236:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829263.231237:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829263.231238:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829263.231239:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829263.231240:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829263.231241:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231242:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231245:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829263.231246:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.231247:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829263.231248:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829263.231249:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829263.231252:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231257:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829263.231258:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.231261:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829263.231262:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829263.231264:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829263.231267:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326976:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829263.231271:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829263.231273:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829263.231275:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829263.231277:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829263.231280:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829263.231284:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829263.231287:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829263.231290:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829263.231294:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829263.231297:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327326976, portal 4
00000100:00000001:1.0:1787829263.231299:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829263.231301:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829263.231307:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231312:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829263.231314:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829263.231316:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829263.231318:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327326976, offset 0
00000400:00000200:1.0:1787829263.231321:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829263.231324:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.231327:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829263.231329:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829263.231331:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829263.231333:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829263.231335:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829263.231337:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40326:40326:256:4294967295] 192.168.203.116@tcp LPNI seq info [40326:40326:8:4294967295]
00000400:00000200:1.0:1787829263.231340:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829263.231344:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829263.231347:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.231349:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829263.231352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829263.231354:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829263.231357:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829263.231359:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829263.231362:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829263.231363:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829263.231365:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829263.231368:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829263.231371:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829263.231372:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829263.231374:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829263.231376:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231382:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.231384:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231388:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.231389:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829263.231391:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829263.231394:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829263.231396:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829263.231398:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298083352
00000100:00000001:1.0:1787829263.231400:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298084013
00000100:00000001:1.0:1787829263.231402:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298083352
00000100:00000001:1.0:1787829263.231404:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298084013
00000100:00000001:1.0:1787829263.231405:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829263.231406:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.231408:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829263.231410:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829263/real 1787829263]  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829263.231414:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829263.231415:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829263.231416:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829263.231418:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829263.231420:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829263.231422:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829263.231426:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829263.231443:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231450:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829263.231452:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231454:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.231456:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829263.231460:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231468:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829263.231472:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231477:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829263.231479:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231480:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829263.231481:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231483:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829263.231484:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829263.231485:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829263.231487:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829263.231489:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829263.231491:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829263.231496:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327326976:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829263.231498:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231499:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829263.231501:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327326976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829263.231511:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829263.231512:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829263.231513:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829263.231515:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829263.231516:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829263.231517:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829263.231518:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829263.231520:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829263.231521:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829263.231522:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829263.231524:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829263.231526:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829263.231526:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231528:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829263.231530:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829263.231531:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829263.231533:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829263.231534:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829263.932803:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298084013, at 4298084032
00000100:00000040:3.0:1787829263.932803:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298084013, at 4298084032
00000100:00000010:2.0:1787829263.932823:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ae0.
00000100:00000010:3.0:1787829263.932826:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f60.
00000020:00000001:2.0:1787829264.060854:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829264.060860:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829264.060864:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829267.132860:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829267.132881:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829267.132884:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829268.348808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829268.348819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829268.348822:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829268.348826:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829268.348832:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829268.348840:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3416
00000400:00000200:1.0:1787829268.348849:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829268.348856:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829268.348860:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829268.348863:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.348865:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.348867:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.348870:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829268.348874:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829268.348876:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829268.348879:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829268.348890:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829268.348892:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829268.348894:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829268.348896:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.348898:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.348900:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.348901:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829268.348903:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829268.348906:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829268.348908:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829268.348910:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829268.348913:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829268.348916:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.348918:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829268.348920:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829268.348932:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829268.348934:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829268.348939:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829268.348940:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.348943:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829268.348946:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829268.348951:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829268.348960:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829268.348968:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829268.348978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327327104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829268.348988:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829268.348992:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829268.348996:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327327104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829268.349023:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829268.349026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829268.349029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829268.349030:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829268.349031:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829268.349034:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829268.349036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3416
00000400:00000200:1.0:1787829268.349040:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829268.349044:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829268.349046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829268.349049:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349051:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.349053:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349057:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829268.349059:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829268.349062:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829268.349065:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829268.349068:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829268.349069:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829268.349072:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829268.349076:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349080:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.349084:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349088:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829268.349093:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829268.349096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349099:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829268.349103:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829268.349112:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829268.349115:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349120:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829268.349126:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829268.349130:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349133:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829268.349140:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829268.349143:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829268.349145:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829268.349146:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349147:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829268.349150:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829268.349153:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829268.349160:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829268.349164:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829268.349171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327327232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829268.349179:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829268.349181:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829268.349186:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327327232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829268.349192:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829268.349195:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829268.349200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829268.349203:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829268.349204:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829268.349208:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829268.349213:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3416
00000400:00000200:1.0:1787829268.349221:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829268.349226:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829268.349230:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829268.349236:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349240:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.349242:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349247:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829268.349249:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829268.349251:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829268.349253:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829268.349256:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829268.349258:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829268.349260:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829268.349262:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349263:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.349266:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349269:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829268.349272:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829268.349274:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349278:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829268.349280:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829268.349284:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829268.349287:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349289:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829268.349293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829268.349297:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349300:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829268.349305:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829268.349308:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829268.349311:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829268.349314:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349317:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829268.349319:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829268.349322:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829268.349327:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829268.349331:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829268.349339:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327327360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829268.349351:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829268.349354:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829268.349360:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327327360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829268.349366:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829268.349370:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829268.349373:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829268.349374:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829268.349375:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829268.349379:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829268.349382:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3416
00000400:00000200:1.0:1787829268.349389:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829268.349392:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829268.349394:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829268.349400:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349402:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.349407:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829268.349408:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829268.349409:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829268.349414:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.1:1787829268.349415:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000020:00000040:1.0:1787829268.349416:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829268.349419:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829268.349422:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829268.349425:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829268.349429:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829268.349434:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349437:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829268.349440:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349442:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829268.349446:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829268.349449:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829268.349452:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829268.349455:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829268.349458:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829268.349461:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349463:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829268.349465:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829268.349468:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349471:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829268.349474:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829268.349478:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829268.349481:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829268.349483:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829268.349487:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829268.349489:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829268.349492:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829268.349497:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829268.349502:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829268.349508:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327327488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829268.349543:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829268.349546:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829268.349551:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327327488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829268.349559:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829268.349564:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3426)
00000800:00000200:0.1:1787829268.349884:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829268.349886:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829268.350088:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829268.350089:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829268.350315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829268.350317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:3.0:1787829269.052793:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298089132, at 4298089152
00000100:00000040:2.0:1787829269.052802:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298089132, at 4298089152
00000100:00000010:3.0:1787829269.052807:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000100:00000010:2.0:1787829269.052813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000020:00000001:2.0:1787829270.204888:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829270.204895:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829270.204901:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829273.276832:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829273.276836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829273.276839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829273.468795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829273.468806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829273.468810:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829273.468815:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829273.468820:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829273.468828:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3421
00000400:00000200:1.0:1787829273.468838:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829273.468846:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829273.468849:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829273.468852:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.468856:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.468858:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.468860:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829273.468864:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829273.468866:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829273.468869:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829273.468872:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829273.468874:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829273.468876:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829273.468878:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.468880:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.468881:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.468883:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829273.468885:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829273.468888:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829273.468890:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829273.468893:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829273.468896:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829273.468898:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.468900:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829273.468902:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829273.468915:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829273.468919:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829273.468925:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829273.468927:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.468931:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829273.468934:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829273.468940:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829273.468953:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829273.468961:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829273.468973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327327616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829273.468984:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829273.468989:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829273.468992:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327327616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829273.469018:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829273.469024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829273.469028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829273.469029:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829273.469031:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829273.469033:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829273.469036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3421
00000400:00000200:1.0:1787829273.469039:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829273.469042:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829273.469046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829273.469050:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469054:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.469056:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469060:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829273.469062:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829273.469063:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829273.469065:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829273.469067:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829273.469070:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829273.469074:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829273.469080:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469084:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.469088:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469091:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829273.469094:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829273.469098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469101:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829273.469103:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829273.469111:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829273.469115:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469118:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829273.469126:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829273.469133:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469137:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829273.469144:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829273.469150:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829273.469157:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829273.469160:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469165:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829273.469168:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829273.469171:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829273.469178:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829273.469184:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829273.469193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327327744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829273.469211:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829273.469213:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829273.469217:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327327744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829273.469242:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829273.469253:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829273.469259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829273.469260:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829273.469264:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829273.469269:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829273.469273:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3421
00000400:00000200:1.0:1787829273.469311:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829273.469316:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829273.469320:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829273.469325:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469331:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.469332:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469336:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829273.469338:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829273.469340:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829273.469343:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829273.469345:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829273.469346:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829273.469348:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829273.469350:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469351:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.469353:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469354:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829273.469359:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829273.469362:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469364:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829273.469366:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829273.469370:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829273.469375:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469377:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829273.469380:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829273.469383:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829273.469386:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829273.469387:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829273.469392:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000800:00000200:0.1:1787829273.469393:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000020:00000040:1.0:1787829273.469396:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829273.469401:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829273.469405:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469407:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829273.469409:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829273.469411:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829273.469414:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829273.469417:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829273.469424:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327327872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829273.469432:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829273.469435:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829273.469442:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327327872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829273.469452:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829273.469457:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829273.469462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829273.469465:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829273.469467:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829273.469470:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829273.469474:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3421
00000400:00000200:1.0:1787829273.469479:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829273.469484:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829273.469488:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829273.469507:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469511:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.469514:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469515:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829273.469517:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829273.469518:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829273.469520:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829273.469523:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829273.469525:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829273.469530:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829273.469535:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469541:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829273.469542:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469545:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829273.469550:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829273.469552:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829273.469554:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829273.469555:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829273.469559:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829273.469562:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469563:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829273.469564:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829273.469566:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469568:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829273.469574:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829273.469577:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829273.469579:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829273.469581:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829273.469584:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829273.469586:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829273.469589:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829273.469595:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829273.469604:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829273.469613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327328000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829273.469635:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829273.469639:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829273.469650:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327328000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829273.469694:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829273.469701:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3431)
00000800:00000200:0.1:1787829273.469912:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829273.469914:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829273.470150:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829273.470151:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829273.470340:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829273.470342:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:3.0:1787829274.172780:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298094252, at 4298094272
00000100:00000010:3.0:1787829274.172795:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000100:00000040:2.0:1787829274.172843:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298094252, at 4298094272
00000100:00000010:2.0:1787829274.172877:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000020:00000001:2.0:1787829276.348901:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829276.348905:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829276.348908:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829278.588808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829278.588821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829278.588824:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829278.588846:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829278.588853:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829278.588864:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3426
00000400:00000200:2.0:1787829278.588876:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829278.588885:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829278.588889:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829278.588893:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.588897:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.588901:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.588904:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829278.588910:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829278.588914:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829278.588917:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829278.588922:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829278.588925:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829278.588928:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829278.588931:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.588934:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.588936:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.588938:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829278.588941:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829278.588945:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829278.588948:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829278.588952:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829278.588956:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829278.588960:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.588962:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829278.588965:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829278.588980:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829278.588984:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829278.588990:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829278.588992:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.588996:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829278.589000:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829278.589007:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829278.589017:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829278.589027:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829278.589039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327328128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829278.589053:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829278.589059:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829278.589066:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327328128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829278.589114:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829278.589135:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829278.589138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829278.589140:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829278.589142:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829278.589146:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829278.589148:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3426
00000400:00000200:2.0:1787829278.589152:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829278.589156:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829278.589159:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829278.589162:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589165:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.589167:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589170:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829278.589172:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829278.589174:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829278.589176:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829278.589179:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829278.589181:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829278.589184:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829278.589188:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589191:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.589194:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589218:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829278.589222:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829278.589230:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589237:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829278.589243:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829278.589253:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829278.589264:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.589271:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829278.589298:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829278.589317:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.589324:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829278.589334:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787829278.589342:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829278.589350:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829278.589355:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.589358:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829278.589362:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829278.589369:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829278.589376:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829278.589403:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829278.589419:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327328256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829278.589437:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829278.589444:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829278.589452:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327328256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829278.589526:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829278.589536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829278.589546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829278.589554:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829278.589561:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829278.589569:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829278.589578:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3426
00000400:00000200:2.0:1787829278.589588:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829278.589599:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829278.589605:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829278.589628:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589634:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.589642:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589650:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829278.589698:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829278.589701:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829278.589705:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829278.589709:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829278.589712:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829278.589715:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829278.589724:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589729:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.589734:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589743:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829278.589751:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829278.589761:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829278.589768:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829278.589775:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829278.589781:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829278.589790:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.589798:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.1:1787829278.589802:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00100000:2.0:1787829278.589806:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.1:1787829278.589808:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000008:00000001:2.0:1787829278.589815:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.589823:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829278.589834:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787829278.589841:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829278.589847:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829278.589852:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.589858:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829278.589861:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829278.589863:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829278.589868:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829278.589877:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829278.589884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327328384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829278.589898:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829278.589903:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829278.589912:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327328384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829278.589934:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829278.589938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829278.589944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829278.589948:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829278.589952:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829278.589961:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829278.589967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3426
00000400:00000200:2.0:1787829278.589988:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829278.589999:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829278.590003:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829278.590007:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.590012:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.590014:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.590017:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829278.590023:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829278.590025:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829278.590027:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829278.590029:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829278.590032:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829278.590037:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829278.590043:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.590048:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829278.590051:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829278.590052:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829278.590054:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829278.590057:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829278.590064:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829278.590069:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829278.590074:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829278.590080:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.590086:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829278.590091:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829278.590098:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.590102:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829278.590107:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000020:00000040:2.0:1787829278.590112:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829278.590117:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829278.590119:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829278.590125:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829278.590127:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829278.590130:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829278.590140:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829278.590146:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829278.590157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327328512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829278.590199:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829278.590201:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829278.590211:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327328512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829278.590231:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829278.590239:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3436)
00000800:00000200:0.1:1787829278.590560:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829278.590562:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829278.590925:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829278.590927:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829278.591222:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829278.591225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1787829279.292787:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298099373, at 4298099392
00000100:00000040:3.0:1787829279.292787:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298099373, at 4298099392
00000100:00000010:3.0:1787829279.292804:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000100:00000010:2.0:1787829279.292807:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000020:00000001:2.0:1787829279.420852:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829279.420856:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829279.420860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829281.148847:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829281.148858:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829281.148861:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829281.148877:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829281.148882:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829281.148887:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829281.148890:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829281.148894:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829281.148897:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829282.172804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829282.172811:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829282.172813:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829282.172819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829282.172823:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829282.172826:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829282.172829:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829282.172832:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829282.172834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829282.492930:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829282.492934:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829282.492937:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829283.196765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829283.196772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829283.196774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829283.196778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829283.196782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829283.196785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829283.196788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829283.196790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829283.196793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829283.708764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829283.708776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829283.708779:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829283.708785:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829283.708791:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829283.708801:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3431
00000400:00000200:3.0:1787829283.708813:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829283.708822:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829283.708826:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829283.708830:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.708833:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.708835:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.708838:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829283.708842:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829283.708844:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829283.708846:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829283.708850:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829283.708852:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829283.708854:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829283.708856:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.708858:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.708859:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.708861:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829283.708863:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829283.708866:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829283.708868:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829283.708870:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829283.708874:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829283.708876:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.708878:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829283.708881:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829283.708891:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829283.708894:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829283.708898:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829283.708899:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.708902:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829283.708904:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829283.708909:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829283.708917:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829283.708923:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829283.708932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327328640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829283.708942:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829283.708946:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829283.708951:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327328640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829283.708995:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829283.708998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829283.709001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829283.709002:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829283.709003:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829283.709006:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829283.709008:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3431
00000400:00000200:3.0:1787829283.709010:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829283.709012:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829283.709014:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829283.709017:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709018:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.709020:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709022:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829283.709023:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829283.709024:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829283.709025:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829283.709027:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829283.709028:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829283.709030:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829283.709032:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709033:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.709034:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709036:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829283.709038:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829283.709040:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709043:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829283.709045:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829283.709051:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829283.709054:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709058:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829283.709066:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829283.709074:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709079:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829283.709086:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829283.709093:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829283.709100:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829283.709104:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709108:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829283.709113:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829283.709117:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829283.709125:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829283.709131:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829283.709138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327328768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829283.709148:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829283.709152:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829283.709158:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327328768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829283.709178:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829283.709184:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829283.709190:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829283.709195:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829283.709200:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829283.709223:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829283.709233:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3431
00000400:00000200:3.0:1787829283.709251:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829283.709261:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829283.709269:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829283.709278:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709283:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.709290:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709296:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829283.709304:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829283.709306:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829283.709309:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829283.709316:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829283.709321:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829283.709327:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829283.709337:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709340:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.709343:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709345:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829283.709353:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829283.709361:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709364:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829283.709371:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829283.709379:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829283.709385:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709390:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829283.709396:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829283.709402:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709406:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829283.709413:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829283.709416:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829283.709420:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829283.709444:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709449:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829283.709471:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829283.709475:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829283.709482:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829283.709489:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829283.709500:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327328896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829283.709518:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829283.709521:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829283.709528:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327328896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829283.709541:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829283.709544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829283.709549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829283.709550:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829283.709552:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829283.709560:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829283.709566:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3431
00000800:00000200:0.1:1787829283.709572:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:3.0:1787829283.709578:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.1:1787829283.709578:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000400:00000200:3.0:1787829283.709588:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829283.709593:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829283.709600:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709604:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.709609:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709612:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829283.709617:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829283.709619:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829283.709622:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829283.709626:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829283.709630:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829283.709633:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829283.709637:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709643:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829283.709650:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709655:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829283.709662:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829283.709702:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829283.709708:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829283.709711:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829283.709715:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829283.709718:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709727:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829283.709729:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829283.709733:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709735:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829283.709739:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829283.709743:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829283.709745:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829283.709748:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829283.709750:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829283.709752:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829283.709755:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829283.709761:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829283.709767:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829283.709776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327329024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829283.709794:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829283.709799:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829283.709809:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327329024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829283.709831:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829283.709838:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3441)
00000800:00000200:0.1:1787829283.710046:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829283.710048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829283.710258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829283.710260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829283.710522:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829283.710524:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1787829284.220777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829284.220789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829284.220809:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829284.220825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829284.220829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829284.220832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829284.220837:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829284.220839:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829284.220842:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829284.412756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298104493, at 4298104512
00000100:00000040:3.0:1787829284.412756:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298104493, at 4298104512
00000100:00000010:2.0:1787829284.412775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000010:3.0:1787829284.412779:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000100:00000001:2.0:1787829285.244767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829285.244779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829285.244782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829285.244791:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829285.244797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829285.244804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829285.244809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829285.244813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829285.244819:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829285.564877:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829285.564881:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829285.564885:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829286.268811:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829286.268820:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829286.268822:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829286.268827:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829286.268831:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829286.268834:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829286.268837:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829286.268839:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829286.268841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829287.292795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829287.292802:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829287.292804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829287.292808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829287.292812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829287.292816:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829287.292818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829287.292821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829287.292823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829288.636899:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829288.636903:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829288.636906:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829288.828741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829288.828750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829288.828752:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829288.828755:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829288.828758:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829288.828780:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3436
00000400:00000200:3.0:1787829288.828791:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829288.828797:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829288.828801:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829288.828804:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828807:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.828809:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828811:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829288.828814:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829288.828815:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829288.828817:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829288.828820:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829288.828822:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829288.828824:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829288.828826:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828827:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.828829:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828830:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829288.828855:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829288.828858:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828860:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829288.828862:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829288.828865:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829288.828867:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.828869:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829288.828871:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829288.828879:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829288.828882:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829288.828885:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829288.828886:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.828889:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829288.828891:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829288.828894:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829288.828900:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829288.828905:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829288.828913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327329152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829288.828920:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829288.828924:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829288.828927:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327329152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829288.828949:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829288.828952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829288.828954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829288.828955:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829288.828956:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829288.828959:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829288.828960:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3436
00000400:00000200:3.0:1787829288.828964:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829288.828966:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829288.828969:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829288.828972:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828974:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.828977:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828979:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829288.828982:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829288.828983:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829288.828985:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829288.828988:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829288.828991:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829288.828993:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829288.828996:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.828999:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.829002:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829005:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829288.829008:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829288.829011:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829014:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829288.829017:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829288.829023:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829288.829030:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829034:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829288.829044:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829288.829053:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829063:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829288.829076:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829288.829082:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829288.829089:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829288.829094:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829097:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829288.829099:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829288.829109:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829288.829115:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829288.829123:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829288.829132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327329280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829288.829143:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829288.829147:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829288.829153:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327329280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829288.829182:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829288.829186:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829288.829193:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829288.829197:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829288.829202:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829288.829205:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829288.829208:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3436
00000400:00000200:3.0:1787829288.829215:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829288.829227:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829288.829231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829288.829241:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829244:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.829250:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829256:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829288.829265:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829288.829269:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829288.829275:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829288.829282:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829288.829294:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829288.829322:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829288.829335:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829341:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.829351:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829358:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829288.829363:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829288.829372:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829376:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829288.829407:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829288.829424:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829288.829428:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829430:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829288.829432:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829288.829436:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829441:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829288.829448:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829288.829453:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829288.829462:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829288.829465:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829467:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829288.829468:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829288.829470:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829288.829477:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829288.829482:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829288.829490:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327329408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829288.829522:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829288.829527:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829288.829544:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327329408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829288.829558:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829288.829570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829288.829579:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829288.829580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829288.829593:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829288.829596:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829288.829600:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829288.829603:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3436
00000400:00000200:3.0:1787829288.829615:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829288.829624:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829288.829628:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829288.829655:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829658:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.829660:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829663:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829288.829665:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829288.829667:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829288.829689:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829288.829693:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829288.829695:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829288.829697:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829288.829702:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829713:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829288.829722:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829734:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829288.829741:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829288.829752:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829288.829768:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829288.829791:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829288.829798:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829288.829801:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829802:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829288.829806:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829288.829821:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829831:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829288.829841:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829288.829848:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829288.829860:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829288.829865:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829288.829893:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829288.829902:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829288.829915:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829288.829929:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829288.829937:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829288.829947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327329536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829288.829962:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829288.829968:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829288.829987:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327329536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829288.830008:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829288.830015:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3446)
00000800:00000200:0.1:1787829288.830139:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829288.830295:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829288.830481:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829289.532757:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298109612, at 4298109632
00000100:00000040:3.0:1787829289.532773:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298109613, at 4298109632
00000100:00000010:2.0:1787829289.532782:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000010:3.0:1787829289.532784:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000020:00000001:2.0:1787829291.708906:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829291.708912:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829291.708934:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1787829291.901083:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:1.0:1787829291.901185:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1787829291.901198:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829291.901203:0:2335:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:1.0:1787829291.901222:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829291.901228:0:2335:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1787829291.901234:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829291.901239:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:1.0:1787829291.901246:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829291.901252:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829291.901260:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefcc0
00000400:00000010:1.0:1787829291.901264:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefcc0.
00000100:00000001:1.0:1787829291.901270:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829291.901282:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0000 x1874671327329152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901312:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829291.901317:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327329152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901325:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829291.901327:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829291.901339:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829291.901341:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:1.0:1787829291.901346:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829291.901350:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829291.901354:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefbb0
00000400:00000010:1.0:1787829291.901357:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefbb0.
00000100:00000001:1.0:1787829291.901360:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829291.901378:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed2d00 x1874671327329280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901392:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829291.901399:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327329280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901411:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829291.901414:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829291.901426:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829291.901434:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000200:1.0:1787829291.901440:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829291.901447:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829291.901455:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefb28
00000400:00000010:1.0:1787829291.901460:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefb28.
00000100:00000001:1.0:1787829291.901466:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829291.901476:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed21c0 x1874671327329408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901492:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829291.901500:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327329408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901517:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829291.901520:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829291.901533:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829291.901539:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000200:1.0:1787829291.901545:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829291.901554:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829291.901559:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef7f8
00000400:00000010:1.0:1787829291.901566:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef7f8.
00000100:00000001:1.0:1787829291.901572:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829291.901581:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0780 x1874671327329536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901613:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829291.901625:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327329536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829291.901648:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829291.901653:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:3.0:1787829293.948793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829293.948801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829293.948803:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829293.948815:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829293.948820:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829293.948826:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3441
00000400:00000200:3.0:1787829293.948835:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829293.948841:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829293.948844:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829293.948848:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.948850:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.948852:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.948856:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829293.948860:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829293.948862:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829293.948864:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829293.948867:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829293.948869:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829293.948871:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829293.948874:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.948875:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.948876:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.948878:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829293.948880:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829293.948882:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829293.948884:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829293.948886:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829293.948888:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829293.948891:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.948893:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829293.948895:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829293.948905:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829293.948909:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829293.948913:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829293.948915:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.948918:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829293.948920:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829293.948925:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829293.948933:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829293.948939:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829293.948949:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327329664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829293.948966:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829293.948970:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829293.948974:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327329664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829293.949000:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829293.949005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829293.949009:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829293.949010:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829293.949013:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829293.949015:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829293.949018:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3441
00000400:00000200:3.0:1787829293.949022:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829293.949025:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829293.949027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829293.949029:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949032:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.949034:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949035:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829293.949038:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829293.949039:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829293.949040:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829293.949043:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829293.949045:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829293.949047:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829293.949050:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949052:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.949054:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949056:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829293.949058:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829293.949061:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949064:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829293.949066:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829293.949072:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829293.949074:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949076:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829293.949080:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829293.949084:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949086:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829293.949091:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829293.949094:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829293.949096:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829293.949098:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949099:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829293.949101:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829293.949103:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829293.949108:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829293.949112:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829293.949118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327329792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829293.949127:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829293.949130:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829293.949135:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327329792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829293.949144:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829293.949147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829293.949150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829293.949153:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829293.949156:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829293.949161:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829293.949167:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3441
00000400:00000200:3.0:1787829293.949174:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829293.949180:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829293.949187:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829293.949195:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949201:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.949205:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949207:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829293.949213:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829293.949217:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829293.949220:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829293.949224:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829293.949230:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829293.949236:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829293.949246:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949251:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.949256:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949262:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829293.949267:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829293.949273:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949281:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829293.949286:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829293.949292:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829293.949298:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949303:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829293.949306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829293.949311:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949313:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829293.949323:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829293.949327:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829293.949334:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829293.949338:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949342:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829293.949345:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829293.949365:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829293.949373:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829293.949382:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829293.949393:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327329920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829293.949406:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829293.949411:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829293.949419:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327329920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829293.949436:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829293.949442:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829293.949450:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829293.949455:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829293.949459:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829293.949464:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829293.949473:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3441
00000400:00000200:3.0:1787829293.949497:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829293.949512:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829293.949517:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829293.949524:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949527:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.949531:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949533:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829293.949536:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829293.949539:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829293.949543:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829293.949547:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:0.1:1787829293.949548:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:3.0:1787829293.949553:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:0.1:1787829293.949556:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000020:00000001:3.0:1787829293.949557:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829293.949568:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949697:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829293.949704:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949711:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829293.949714:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829293.949722:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829293.949726:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829293.949731:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829293.949735:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829293.949741:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949746:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829293.949749:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829293.949754:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949760:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829293.949769:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829293.949776:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829293.949782:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829293.949786:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829293.949790:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829293.949792:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829293.949794:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829293.949799:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829293.949806:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829293.949817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327330048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829293.949833:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829293.949837:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829293.949849:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327330048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829293.949866:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829293.949875:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3451)
00000800:00000200:0.1:1787829293.950093:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829293.950095:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829293.950505:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829293.950508:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829293.950870:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829293.950873:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1787829294.652714:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298114733, at 4298114752
00000100:00000010:3.0:1787829294.652731:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000100:00000040:2.0:1787829294.652792:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298114733, at 4298114752
00000100:00000010:2.0:1787829294.652808:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000020:00000001:2.0:1787829294.780895:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829294.780900:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829294.780904:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829297.852906:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829297.852912:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829297.852916:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829299.068828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829299.068850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829299.068853:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829299.068873:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829299.068878:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829299.068886:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3446
00000400:00000200:1.0:1787829299.068912:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829299.068919:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829299.068922:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829299.068941:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.068944:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.068946:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.068948:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829299.068952:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829299.068954:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829299.068957:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829299.068960:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829299.068962:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829299.068964:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829299.068967:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.068968:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.068971:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.068972:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829299.068986:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829299.068989:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829299.068992:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829299.068994:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829299.068997:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829299.069000:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069002:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829299.069004:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829299.069030:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829299.069032:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829299.069036:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829299.069037:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069040:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829299.069043:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829299.069048:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829299.069056:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829299.069063:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829299.069073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327330176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829299.069082:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829299.069085:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829299.069089:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327330176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829299.069108:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829299.069110:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829299.069113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829299.069114:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829299.069116:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829299.069118:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829299.069120:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3446
00000400:00000200:1.0:1787829299.069123:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829299.069125:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829299.069127:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829299.069129:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069130:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.069132:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069134:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829299.069135:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829299.069136:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829299.069137:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829299.069139:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829299.069140:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829299.069142:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829299.069144:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069146:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.069148:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069150:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829299.069151:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829299.069153:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069155:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829299.069156:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829299.069191:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829299.069195:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069200:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829299.069206:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829299.069211:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069213:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829299.069219:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829299.069223:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829299.069229:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829299.069232:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069237:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829299.069241:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829299.069245:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829299.069258:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829299.069265:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829299.069274:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327330304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829299.069287:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829299.069309:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829299.069329:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327330304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829299.069352:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829299.069358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829299.069364:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829299.069384:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829299.069386:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829299.069388:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829299.069394:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3446
00000400:00000200:1.0:1787829299.069401:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829299.069403:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829299.069413:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829299.069417:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069420:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.069421:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069424:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829299.069429:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829299.069431:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829299.069433:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829299.069454:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829299.069455:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829299.069458:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829299.069463:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069468:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.069470:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069472:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829299.069480:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829299.069483:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069485:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829299.069486:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829299.069490:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829299.069494:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069496:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829299.069498:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829299.069500:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069517:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829299.069524:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829299.069530:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829299.069536:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829299.069538:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069551:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829299.069553:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829299.069557:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829299.069564:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829299.069579:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829299.069606:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327330432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829299.069627:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829299.069631:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829299.069638:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327330432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829299.069647:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829299.069653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829299.069659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829299.069661:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829299.069663:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829299.069666:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829299.069779:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3446
00000400:00000200:1.0:1787829299.069785:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829299.069790:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829299.069792:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829299.069794:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069797:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.069803:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069806:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829299.069810:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829299.069814:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829299.069819:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829299.069823:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829299.069826:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829299.069831:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829299.069845:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069849:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829299.069851:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069853:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829299.069855:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829299.069861:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829299.069863:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829299.069864:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829299.069883:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829299.069884:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069886:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829299.069889:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829299.069891:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069892:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829299.069895:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829299.069897:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829299.069899:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829299.069901:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829299.069902:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829299.069903:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829299.069905:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829299.069909:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829299.069912:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829299.069920:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327330560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829299.069929:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829299.069949:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829299.069955:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327330560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829299.069967:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829299.069972:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3456)
00000800:00000200:0.1:1787829299.069972:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829299.069981:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829299.070668:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829299.070672:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829299.071154:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829299.071184:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829299.071634:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829299.071637:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:3.0:1787829299.772722:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298119853, at 4298119872
00000100:00000040:2.0:1787829299.772724:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298119853, at 4298119872
00000100:00000010:3.0:1787829299.772738:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000100:00000010:2.0:1787829299.772740:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000020:00000001:2.0:1787829300.924826:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829300.924834:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829300.924839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829302.652819:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829302.652825:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829302.652827:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829302.652832:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829302.652836:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829302.652838:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829302.652841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829302.652842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829302.652845:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829303.676747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829303.676753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829303.676755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829303.676759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829303.676772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829303.676775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829303.676777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829303.676779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829303.676781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829303.996819:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829303.996824:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829303.996828:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829304.188801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829304.188812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829304.188815:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829304.188819:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829304.188825:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829304.188833:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3451
00000400:00000200:1.0:1787829304.188843:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829304.188850:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829304.188853:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829304.188855:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.188858:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.188861:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.188863:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829304.188867:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829304.188869:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829304.188871:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829304.188875:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829304.188877:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829304.188878:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829304.188880:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.188882:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.188883:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.188884:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829304.188887:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829304.188890:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829304.188892:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829304.188895:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829304.188899:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829304.188902:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.188904:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829304.188907:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829304.188919:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829304.188923:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829304.188928:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829304.188930:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.188934:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829304.188938:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829304.188943:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829304.188953:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829304.188970:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829304.188982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327330688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829304.188994:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829304.189000:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829304.189006:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327330688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829304.189029:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829304.189033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829304.189035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829304.189036:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829304.189038:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829304.189040:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829304.189042:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3451
00000400:00000200:1.0:1787829304.189045:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829304.189047:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829304.189049:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829304.189052:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189055:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.189058:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189061:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829304.189063:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829304.189064:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829304.189065:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829304.189067:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829304.189069:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829304.189070:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829304.189073:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189078:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.189081:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189085:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829304.189092:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829304.189097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189107:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829304.189113:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829304.189122:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829304.189129:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189135:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829304.189141:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829304.189153:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189165:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829304.189173:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829304.189181:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829304.189187:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829304.189196:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189211:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829304.189223:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829304.189225:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829304.189229:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829304.189236:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829304.189246:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327330816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829304.189267:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829304.189269:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829304.189275:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327330816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829304.189300:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829304.189306:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829304.189337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829304.189342:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829304.189346:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829304.189355:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829304.189360:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3451
00000400:00000200:1.0:1787829304.189376:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829304.189387:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829304.189400:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829304.189415:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189431:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.189437:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189444:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829304.189448:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829304.189450:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829304.189452:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829304.189455:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829304.189461:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829304.189465:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829304.189469:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189475:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.189482:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189489:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829304.189492:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829304.189501:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189505:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829304.189508:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829304.189515:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829304.189520:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189524:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829304.189528:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829304.189534:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189538:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829304.189544:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829304.189551:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829304.189561:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829304.189565:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189569:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829304.189575:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829304.189578:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829304.189587:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829304.189600:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829304.189613:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327330944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829304.189629:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829304.189632:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829304.189641:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327330944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829304.189662:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829304.189666:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829304.189697:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829304.189700:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00080000:1.0:1787829304.189707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829304.189728:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829304.189734:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829304.189746:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829304.189752:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3451
00000400:00000200:1.0:1787829304.189762:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829304.189768:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829304.189772:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829304.189782:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189797:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.189802:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189812:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829304.189820:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829304.189826:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829304.189833:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829304.189837:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829304.189839:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829304.189844:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829304.189857:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189865:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829304.189869:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189876:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829304.189879:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829304.189882:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829304.189892:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829304.189896:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829304.189904:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829304.189916:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189919:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829304.189925:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829304.189929:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189930:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829304.189935:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829304.189949:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829304.189962:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829304.189968:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829304.189973:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829304.189979:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829304.189984:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829304.189996:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829304.190005:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829304.190014:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327331072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829304.190038:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829304.190043:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829304.190054:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327331072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829304.190125:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829304.190143:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3461)
00000800:00000200:0.1:1787829304.190446:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829304.190459:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829304.190693:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829304.190696:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829304.190899:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829304.190901:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829304.700786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829304.700794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829304.700797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829304.700802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829304.700806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829304.700810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829304.700813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829304.700817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829304.700820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829304.892708:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298124973, at 4298124992
00000100:00000040:2.0:1787829304.892721:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298124973, at 4298124992
00000100:00000010:3.0:1787829304.892723:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000100:00000010:2.0:1787829304.892732:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000001:2.0:1787829305.724724:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829305.724731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829305.724733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829305.724739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829305.724742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829305.724746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829305.724749:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829305.724751:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829305.724754:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829306.748790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829306.748797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829306.748799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829306.748805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829306.748809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829306.748813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829306.748817:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829306.748820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829306.748823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829307.068917:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829307.068926:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829307.068933:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829307.772720:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829307.772726:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829307.772728:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829307.772731:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829307.772734:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829307.772736:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829307.772739:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829307.772741:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829307.772742:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829308.796771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829308.796777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829308.796779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829308.796783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829308.796786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829308.796789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829308.796792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829308.796794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829308.796797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829309.308767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829309.308775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829309.308777:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829309.308781:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829309.308785:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829309.308796:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3456
00000400:00000200:1.0:1787829309.308805:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829309.308810:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829309.308812:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829309.308815:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308817:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.308819:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308821:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829309.308823:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829309.308825:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829309.308827:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829309.308830:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829309.308832:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829309.308834:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829309.308836:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308838:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.308839:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308841:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829309.308843:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829309.308845:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308847:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829309.308849:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829309.308851:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829309.308853:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.308854:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829309.308856:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829309.308864:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829309.308867:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829309.308870:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829309.308871:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.308874:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829309.308876:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829309.308879:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829309.308886:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829309.308891:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829309.308898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327331200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829309.308906:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829309.308909:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829309.308913:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327331200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829309.308937:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829309.308940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829309.308942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829309.308943:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829309.308944:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829309.308949:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829309.308951:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3456
00000400:00000200:1.0:1787829309.308953:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829309.308955:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829309.308957:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829309.308959:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308961:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.308963:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308966:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829309.308969:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829309.308970:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829309.308973:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829309.308976:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829309.308978:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829309.308981:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829309.308984:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308987:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.308993:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.308998:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829309.309003:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829309.309011:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309017:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829309.309020:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829309.309027:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829309.309030:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309033:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829309.309039:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829309.309044:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309047:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829309.309051:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829309.309055:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829309.309059:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829309.309061:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309062:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829309.309064:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829309.309066:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829309.309072:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829309.309079:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829309.309086:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327331328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829309.309095:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829309.309098:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829309.309104:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327331328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829309.309116:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829309.309121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829309.309128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829309.309131:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829309.309135:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829309.309139:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829309.309145:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3456
00000400:00000200:1.0:1787829309.309154:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829309.309160:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829309.309162:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829309.309166:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309168:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.309172:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309174:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829309.309178:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829309.309180:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829309.309182:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829309.309185:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829309.309188:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829309.309191:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829309.309197:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309202:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.309206:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309211:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829309.309215:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829309.309221:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309224:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829309.309226:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829309.309229:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829309.309233:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309234:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829309.309236:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829309.309237:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309238:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829309.309242:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829309.309259:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829309.309262:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829309.309264:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309266:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829309.309270:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829309.309271:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829309.309275:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829309.309280:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829309.309286:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327331456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829309.309296:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829309.309299:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:0.1:1787829309.309301:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1787829309.309302:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829309.309310:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327331456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829309.309322:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829309.309327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829309.309333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829309.309335:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829309.309337:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829309.309341:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829309.309347:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3456
00000400:00000200:1.0:1787829309.309353:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829309.309359:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829309.309362:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829309.309368:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309371:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.309374:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309377:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829309.309380:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829309.309382:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829309.309384:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829309.309387:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829309.309391:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829309.309395:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829309.309400:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309404:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829309.309407:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309409:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829309.309414:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829309.309418:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829309.309422:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829309.309424:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829309.309428:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829309.309431:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309434:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829309.309436:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829309.309437:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309439:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829309.309445:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829309.309447:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829309.309450:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829309.309452:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829309.309456:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829309.309458:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829309.309460:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829309.309467:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829309.309470:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829309.309474:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327331584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829309.309489:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829309.309493:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829309.309501:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327331584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829309.309512:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829309.309517:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3467)
00000800:00000200:0.1:1787829309.309758:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829309.309761:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829309.309996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829309.309998:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829309.310223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829309.310225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:2.0:1787829309.820742:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829309.820751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829309.820753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829309.820758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829309.820763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829309.820768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829309.820773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829309.820777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829309.820780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829310.012734:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298130092, at 4298130112
00000100:00000040:2.0:1787829310.012743:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298130092, at 4298130112
00000100:00000010:3.0:1787829310.012748:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000100:00000010:2.0:1787829310.012754:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000020:00000001:2.0:1787829310.140814:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829310.140818:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829310.140821:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829310.844763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829310.844771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829310.844773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829310.844779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829310.844783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829310.844788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829310.844791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829310.844794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829310.844797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829311.868792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829311.868798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829311.868799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829311.868803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829311.868806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829311.868809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829311.868811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829311.868813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829311.868816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829312.892795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829312.892804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829312.892807:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829312.892814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829312.892818:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829312.892823:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829312.892826:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829312.892830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829312.892834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829313.212861:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829313.212866:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829313.212870:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829313.916774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829313.916782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829313.916785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829313.916792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829313.916797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829313.916802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829313.916805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829313.916808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829313.916812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829314.428741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829314.428752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829314.428755:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829314.428760:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829314.428765:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829314.428773:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3462
00000400:00000200:1.0:1787829314.428784:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829314.428791:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829314.428795:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829314.428798:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428801:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.428804:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428806:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829314.428810:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829314.428811:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829314.428813:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829314.428817:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829314.428819:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829314.428821:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829314.428823:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428824:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.428826:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428828:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829314.428831:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829314.428834:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428842:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829314.428845:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829314.428848:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829314.428851:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.428853:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829314.428855:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829314.428865:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829314.428868:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829314.428873:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829314.428874:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.428877:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829314.428880:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829314.428884:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829314.428893:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829314.428900:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829314.428910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327331712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829314.428919:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829314.428923:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829314.428927:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327331712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829314.428942:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829314.428944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829314.428947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829314.428948:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829314.428949:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829314.428951:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829314.428954:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3462
00000400:00000200:1.0:1787829314.428958:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829314.428960:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829314.428962:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829314.428965:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428967:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.428969:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428970:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829314.428974:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829314.428976:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829314.428978:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829314.428982:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829314.428984:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829314.428988:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829314.428993:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.428998:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.429001:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429006:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829314.429010:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829314.429014:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429018:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829314.429022:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829314.429028:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829314.429037:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429043:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829314.429051:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829314.429057:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429060:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829314.429069:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829314.429076:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829314.429083:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829314.429088:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429092:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829314.429094:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829314.429097:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829314.429106:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829314.429111:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829314.429121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327331840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829314.429136:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829314.429140:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829314.429146:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327331840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829314.429157:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829314.429163:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829314.429166:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829314.429168:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829314.429171:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829314.429178:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829314.429182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3462
00000400:00000200:1.0:1787829314.429246:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829314.429255:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829314.429263:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829314.429270:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429273:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.429276:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429281:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829314.429287:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829314.429288:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829314.429292:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829314.429295:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:0.1:1787829314.429299:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829314.429300:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829314.429306:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829314.429308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:1.0:1787829314.429311:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429316:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.429320:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429324:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829314.429330:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829314.429337:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429341:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829314.429348:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829314.429352:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829314.429357:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429361:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829314.429365:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829314.429373:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429378:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829314.429385:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829314.429391:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829314.429398:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829314.429402:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429408:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829314.429411:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829314.429415:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829314.429421:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829314.429427:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829314.429435:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327331968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829314.429446:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829314.429448:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829314.429452:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327331968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829314.429464:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829314.429469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829314.429472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829314.429473:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829314.429474:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829314.429476:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829314.429478:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3462
00000400:00000200:1.0:1787829314.429483:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829314.429495:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829314.429499:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829314.429502:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429506:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.429511:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429516:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829314.429520:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829314.429522:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829314.429525:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829314.429528:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829314.429531:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829314.429533:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829314.429538:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429541:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829314.429546:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429548:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829314.429553:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829314.429568:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829314.429578:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829314.429585:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829314.429592:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829314.429599:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429606:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829314.429610:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829314.429615:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429620:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829314.429626:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829314.429631:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829314.429637:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829314.429640:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829314.429643:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829314.429646:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829314.429650:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829314.429691:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829314.429699:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829314.429711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327332096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829314.429731:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829314.429737:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829314.429748:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327332096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829314.429769:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829314.429777:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3472)
00000800:00000200:0.1:1787829314.429932:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829314.429935:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829314.430156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829314.430158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829314.430390:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829314.430392:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1787829314.940764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829314.940771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829314.940774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829314.940779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829314.940783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829314.940787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829314.940789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829314.940791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829314.940794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829315.132771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298135212, at 4298135232
00000100:00000040:3.0:1787829315.132779:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298135212, at 4298135232
00000100:00000010:2.0:1787829315.132794:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000010:3.0:1787829315.132798:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00000001:2.0:1787829315.964797:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829315.964807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829315.964809:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829315.964817:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829315.964822:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829315.964827:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829315.964830:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829315.964834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829315.964838:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829316.284853:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829316.284857:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829316.284861:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829316.988756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829316.988768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829316.988771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829316.988779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829316.988784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829316.988788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829316.988791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829316.988795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829316.988802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829318.012798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829318.012806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829318.012808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829318.012813:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829318.012817:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829318.012822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829318.012825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829318.012827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829318.012830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829319.036761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829319.036771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829319.036774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829319.036781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829319.036786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829319.036791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829319.036795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829319.036799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829319.036803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829319.356854:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829319.356860:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829319.356865:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829319.548765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829319.548774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829319.548776:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829319.548779:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829319.548783:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829319.548790:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3467
00000400:00000200:3.0:1787829319.548799:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829319.548806:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829319.548809:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829319.548814:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548817:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.548820:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548822:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829319.548827:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829319.548829:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829319.548831:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829319.548835:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829319.548837:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829319.548839:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829319.548841:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548843:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.548844:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548845:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829319.548848:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829319.548850:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548852:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829319.548854:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829319.548856:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829319.548859:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.548860:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829319.548862:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829319.548868:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829319.548871:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829319.548874:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829319.548875:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.548878:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829319.548880:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829319.548883:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829319.548888:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829319.548893:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829319.548901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327332224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829319.548909:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829319.548912:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829319.548916:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327332224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829319.548934:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829319.548936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829319.548939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829319.548940:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829319.548941:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829319.548943:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829319.548945:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3467
00000400:00000200:3.0:1787829319.548948:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829319.548950:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829319.548951:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829319.548953:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548955:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.548957:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548959:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829319.548961:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829319.548962:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829319.548963:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829319.548965:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829319.548966:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829319.548968:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829319.548970:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548972:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.548973:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548974:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829319.548975:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829319.548978:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829319.548982:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829319.548984:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829319.548988:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829319.548991:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.548994:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829319.548997:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829319.549000:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549002:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829319.549007:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829319.549010:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829319.549016:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829319.549018:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549021:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829319.549023:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829319.549026:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829319.549031:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829319.549035:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829319.549043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327332352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829319.549062:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829319.549066:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829319.549072:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327332352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829319.549085:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829319.549089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829319.549095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829319.549097:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829319.549099:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829319.549104:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829319.549106:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3467
00000400:00000200:3.0:1787829319.549109:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829319.549115:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829319.549119:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829319.549125:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549129:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.549133:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549138:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829319.549141:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829319.549143:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829319.549146:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829319.549148:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829319.549151:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829319.549154:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829319.549161:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549164:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.549168:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549170:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829319.549184:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829319.549189:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549193:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829319.549195:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829319.549197:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829319.549202:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549205:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829319.549209:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829319.549217:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549222:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829319.549227:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829319.549233:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829319.549237:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829319.549238:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549241:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829319.549244:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829319.549247:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829319.549253:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829319.549260:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829319.549266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327332480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829319.549281:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829319.549284:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829319.549291:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327332480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829319.549291:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829319.549296:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:3.0:1787829319.549306:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829319.549311:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829319.549315:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829319.549318:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829319.549321:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829319.549325:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829319.549330:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3467
00000400:00000200:3.0:1787829319.549337:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829319.549342:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829319.549345:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829319.549351:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549356:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.549359:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549363:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829319.549367:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829319.549371:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829319.549373:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829319.549377:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829319.549382:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829319.549388:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829319.549394:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549396:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829319.549400:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549405:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829319.549409:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829319.549415:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829319.549418:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829319.549420:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829319.549423:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829319.549425:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549429:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829319.549431:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829319.549433:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549438:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829319.549442:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829319.549447:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829319.549450:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829319.549453:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829319.549457:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829319.549460:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829319.549462:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829319.549470:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829319.549476:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829319.549486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327332608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829319.549500:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829319.549505:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829319.549514:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327332608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829319.549530:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829319.549535:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3477)
00000800:00000200:0.1:1787829319.549735:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829319.549747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829319.550006:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829319.550009:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829319.550214:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829319.550216:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787829320.060805:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829320.060814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829320.060817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829320.060825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829320.060829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829320.060834:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829320.060838:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829320.060842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829320.060845:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829320.252746:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298140332, at 4298140352
00000100:00000040:3.0:1787829320.252746:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298140332, at 4298140352
00000100:00000010:2.0:1787829320.252764:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:3.0:1787829320.252767:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000100:00000001:2.0:1787829321.084711:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829321.084718:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829321.084720:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829321.084725:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829321.084729:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829321.084732:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829321.084734:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829321.084737:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829321.084739:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829322.108797:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829322.108805:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829322.108807:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829322.108812:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829322.108816:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829322.108821:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829322.108823:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829322.108826:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829322.108828:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829322.428797:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829322.428802:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829322.428806:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829323.132720:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829323.132727:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829323.132729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829323.132734:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829323.132737:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829323.132741:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829323.132743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829323.132745:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829323.132747:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829324.156773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829324.156783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829324.156786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829324.156806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829324.156811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829324.156817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829324.156821:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829324.156824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829324.156828:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829324.668760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829324.668771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829324.668774:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829324.668778:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829324.668783:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829324.668790:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3472
00000400:00000200:3.0:1787829324.668800:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829324.668806:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829324.668809:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829324.668812:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668815:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.668817:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668819:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829324.668822:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829324.668824:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829324.668827:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829324.668831:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829324.668833:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829324.668835:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829324.668838:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668839:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.668840:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668842:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829324.668845:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829324.668847:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668849:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829324.668858:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829324.668862:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829324.668864:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.668866:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829324.668868:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829324.668878:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829324.668881:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829324.668885:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829324.668887:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.668890:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829324.668893:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829324.668897:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829324.668905:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829324.668911:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829324.668921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327332736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829324.668931:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829324.668935:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829324.668938:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327332736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829324.668957:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829324.668960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829324.668963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829324.668964:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829324.668965:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829324.668967:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829324.668969:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3472
00000400:00000200:3.0:1787829324.668971:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829324.668973:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829324.668975:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829324.668977:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668978:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.668980:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668982:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829324.668984:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829324.668985:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829324.668986:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829324.668987:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829324.668988:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829324.668990:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829324.668992:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668994:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.668997:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.668999:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829324.669001:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829324.669003:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669004:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829324.669006:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829324.669011:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829324.669013:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669014:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829324.669019:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829324.669023:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669027:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829324.669032:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829324.669035:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829324.669040:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829324.669049:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669051:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829324.669053:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829324.669057:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829324.669063:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829324.669073:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829324.669088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327332864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829324.669109:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829324.669112:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829324.669125:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327332864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829324.669166:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829324.669172:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829324.669175:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829324.669176:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829324.669179:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829324.669193:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829324.669202:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3472
00000400:00000200:3.0:1787829324.669216:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829324.669222:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829324.669225:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829324.669237:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669241:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.669249:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669263:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829324.669268:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829324.669271:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829324.669279:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829324.669285:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829324.669294:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829324.669302:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829324.669314:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669319:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.669328:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669334:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829324.669341:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829324.669346:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669350:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829324.669353:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829324.669358:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829324.669363:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669373:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829324.669382:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829324.669396:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669402:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829324.669411:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829324.669419:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829324.669425:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829324.669427:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669430:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829324.669434:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829324.669439:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829324.669454:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829324.669464:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829324.669478:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327332992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829324.669494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829324.669501:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:3.0:1787829324.669532:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829324.669540:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829324.669557:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327332992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829324.669612:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829324.669621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829324.669633:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829324.669637:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829324.669642:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829324.669645:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829324.669677:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3472
00000400:00000200:3.0:1787829324.669682:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829324.669685:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829324.669687:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829324.669692:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669694:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.669695:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669697:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829324.669699:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829324.669700:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829324.669702:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829324.669703:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829324.669706:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829324.669713:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829324.669718:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669727:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829324.669731:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669737:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829324.669742:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829324.669750:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829324.669763:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829324.669768:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829324.669773:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829324.669779:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669785:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829324.669788:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829324.669793:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669804:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829324.669817:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829324.669822:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829324.669834:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829324.669842:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829324.669846:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829324.669849:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829324.669858:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829324.669864:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829324.669873:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829324.669891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327333120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829324.669947:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829324.669951:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829324.669958:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327333120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829324.669975:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829324.669984:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3482)
00000800:00000200:0.1:1787829324.670204:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829324.670206:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829324.670397:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829324.670398:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829324.670605:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829324.670607:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1787829325.180739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829325.180747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829325.180749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829325.180754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829325.180758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829325.180761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829325.180763:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829325.180766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829325.180769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829325.372770:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298145453, at 4298145472
00000100:00000040:2.0:1787829325.372779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298145453, at 4298145472
00000100:00000010:3.0:1787829325.372784:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000100:00000010:2.0:1787829325.372800:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000020:00000001:2.0:1787829325.500833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829325.500840:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829325.500845:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829326.204780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829326.204789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829326.204791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829326.204798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829326.204812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829326.204817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829326.204820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829326.204824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829326.204828:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829327.228734:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829327.228744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829327.228747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829327.228765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829327.228770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829327.228775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829327.228778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829327.228781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829327.228785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829328.252745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829328.252756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829328.252759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829328.252767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829328.252772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829328.252777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829328.252780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829328.252783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829328.252787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829328.572874:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829328.572877:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829328.572880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829329.276747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829329.276756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829329.276759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829329.276766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829329.276771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829329.276776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829329.276780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829329.276783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829329.276787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829329.788744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829329.788754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829329.788758:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829329.788762:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829329.788766:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829329.788774:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3477
00000400:00000200:3.0:1787829329.788783:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829329.788789:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829329.788793:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829329.788797:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788800:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.788803:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788806:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829329.788810:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829329.788811:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829329.788813:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829329.788816:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829329.788818:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829329.788820:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829329.788821:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788823:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.788824:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788825:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829329.788827:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829329.788829:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788831:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829329.788833:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829329.788836:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829329.788838:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.788839:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829329.788841:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829329.788848:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829329.788851:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829329.788855:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829329.788856:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.788859:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829329.788861:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829329.788864:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829329.788870:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829329.788875:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829329.788882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327333248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829329.788890:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829329.788893:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829329.788897:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327333248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829329.788915:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829329.788917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829329.788919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829329.788920:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829329.788921:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829329.788924:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829329.788925:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3477
00000400:00000200:3.0:1787829329.788928:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829329.788930:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829329.788931:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829329.788934:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788936:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.788939:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788942:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829329.788945:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829329.788946:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829329.788948:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829329.788950:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829329.788954:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829329.788959:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829329.788964:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788969:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.788976:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788980:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829329.788986:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829329.788993:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829329.788999:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829329.789004:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829329.789011:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829329.789018:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789024:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829329.789029:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829329.789034:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789036:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829329.789041:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829329.789045:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829329.789050:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829329.789053:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789055:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829329.789056:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829329.789060:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829329.789065:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829329.789070:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829329.789078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327333376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829329.789089:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829329.789093:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829329.789099:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327333376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829329.789142:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829329.789146:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829329.789153:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829329.789157:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829329.789159:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829329.789161:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829329.789165:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3477
00000400:00000200:3.0:1787829329.789172:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829329.789178:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829329.789182:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829329.789186:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789190:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.789195:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789199:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829329.789203:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829329.789206:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829329.789208:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829329.789211:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829329.789216:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829329.789220:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829329.789223:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789227:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.789232:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789236:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829329.789239:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829329.789246:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789249:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829329.789254:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829329.789257:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829329.789261:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789267:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829329.789270:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.1:1787829329.789274:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:3.0:1787829329.789276:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829329.789278:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:3.0:1787829329.789280:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829329.789289:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829329.789294:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829329.789301:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829329.789305:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789311:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829329.789315:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829329.789320:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829329.789329:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829329.789336:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829329.789349:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327333504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829329.789363:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829329.789367:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829329.789371:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327333504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829329.789379:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829329.789382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829329.789386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829329.789388:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829329.789391:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829329.789394:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829329.789395:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3477
00000400:00000200:3.0:1787829329.789398:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829329.789402:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829329.789405:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829329.789411:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789416:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.789420:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789426:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829329.789430:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829329.789432:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829329.789436:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829329.789439:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829329.789443:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829329.789446:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829329.789452:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789455:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829329.789459:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789462:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829329.789466:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829329.789472:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829329.789476:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829329.789480:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829329.789485:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829329.789489:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789493:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829329.789497:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829329.789501:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789505:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829329.789509:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829329.789513:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829329.789519:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829329.789521:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829329.789524:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829329.789527:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829329.789529:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829329.789533:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829329.789540:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829329.789550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327333632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829329.789574:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829329.789580:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829329.789590:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327333632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829329.789605:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829329.789613:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3487)
00000800:00000200:0.1:1787829329.789761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829329.789763:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829329.790017:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829329.790019:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829329.790226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829329.790227:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787829330.300755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829330.300762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829330.300764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829330.300769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829330.300780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829330.300784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829330.300786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829330.300788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829330.300791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829330.492744:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298150572, at 4298150592
00000100:00000040:3.0:1787829330.492744:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298150572, at 4298150592
00000100:00000010:3.0:1787829330.492762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:2.0:1787829330.492763:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000001:2.0:1787829331.324792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829331.324799:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829331.324801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829331.324807:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829331.324811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829331.324814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829331.324817:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829331.324820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829331.324823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829331.644863:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829331.644868:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829331.644872:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829332.348784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829332.348793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829332.348795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829332.348800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829332.348804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829332.348808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829332.348811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829332.348814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829332.348816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829333.372796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829333.372804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829333.372807:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829333.372812:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829333.372816:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829333.372819:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829333.372822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829333.372824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829333.372827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829334.396798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829334.396805:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829334.396807:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829334.396812:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829334.396815:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829334.396819:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829334.396822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829334.396824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829334.396827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829334.716808:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829334.716811:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829334.716814:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829334.908784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829334.908795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829334.908798:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829334.908802:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829334.908807:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829334.908814:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3482
00000400:00000200:1.0:1787829334.908824:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829334.908830:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829334.908833:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829334.908836:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908838:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.908841:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829334.908847:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829334.908849:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829334.908851:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829334.908855:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829334.908857:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829334.908858:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829334.908860:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908861:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.908862:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908864:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829334.908866:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829334.908868:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908870:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829334.908873:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829334.908877:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829334.908879:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.908881:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829334.908883:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829334.908893:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829334.908895:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829334.908899:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829334.908901:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.908904:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829334.908907:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829334.908911:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829334.908919:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829334.908926:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829334.908935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327333760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829334.908945:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829334.908950:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829334.908953:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327333760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829334.908970:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829334.908973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829334.908975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829334.908976:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829334.908977:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829334.908979:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829334.908981:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3482
00000400:00000200:1.0:1787829334.908983:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829334.908985:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829334.908987:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829334.908989:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908991:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.908993:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.908995:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829334.908997:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829334.908998:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829334.908999:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829334.909000:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829334.909002:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829334.909004:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829334.909009:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909011:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.909016:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909018:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829334.909021:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829334.909027:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909033:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829334.909036:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829334.909044:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829334.909047:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909051:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829334.909057:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829334.909063:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909080:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829334.909087:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829334.909092:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829334.909095:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829334.909098:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909103:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829334.909109:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829334.909113:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829334.909122:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829334.909128:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829334.909135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327333888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829334.909144:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829334.909146:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829334.909152:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327333888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829334.909160:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829334.909165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829334.909171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829334.909173:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829334.909176:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829334.909179:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829334.909182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3482
00000400:00000200:1.0:1787829334.909186:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829334.909192:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829334.909196:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829334.909202:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909206:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.909209:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909212:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829334.909217:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829334.909218:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829334.909221:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829334.909223:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829334.909227:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829334.909229:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829334.909238:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909243:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.909248:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909254:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829334.909260:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829334.909266:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909273:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829334.909276:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829334.909282:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829334.909285:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909288:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829334.909293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829334.909297:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909299:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829334.909304:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829334.909309:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829334.909313:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829334.909315:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909316:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829334.909320:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829334.909323:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829334.909329:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829334.909333:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829334.909339:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327334016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829334.909357:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829334.909362:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829334.909368:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327334016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829334.909376:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829334.909379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829334.909383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829334.909386:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829334.909389:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1787829334.909393:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829334.909394:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1787829334.909399:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00080000:1.0:1787829334.909400:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3482
00000400:00000200:1.0:1787829334.909410:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829334.909417:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829334.909420:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829334.909423:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909425:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.909428:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909432:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829334.909433:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829334.909436:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829334.909439:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829334.909441:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829334.909446:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829334.909447:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829334.909452:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909455:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829334.909457:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909462:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829334.909464:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829334.909467:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829334.909471:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829334.909473:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829334.909475:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829334.909477:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909480:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829334.909482:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829334.909486:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909489:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829334.909494:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829334.909498:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829334.909501:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829334.909502:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829334.909504:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829334.909507:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829334.909509:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829334.909513:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829334.909515:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829334.909519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327334144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829334.909525:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829334.909528:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829334.909534:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327334144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829334.909553:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829334.909562:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3492)
00000800:00000200:0.1:1787829334.910019:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829334.910036:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829334.910265:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829334.910267:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829334.910511:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829334.910514:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1787829335.420829:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829335.420838:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829335.420841:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829335.420849:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829335.420854:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829335.420859:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829335.420862:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829335.420865:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829335.420870:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829335.612727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298155692, at 4298155712
00000100:00000040:3.0:1787829335.612727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298155692, at 4298155712
00000100:00000010:3.0:1787829335.612746:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000100:00000010:2.0:1787829335.612751:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000100:00000001:2.0:1787829336.444800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829336.444808:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829336.444810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829336.444817:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829336.444821:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829336.444825:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829336.444827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829336.444830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829336.444833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829337.468795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829337.468802:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829337.468804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829337.468810:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829337.468813:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829337.468818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829337.468820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829337.468823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829337.468826:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829337.788803:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829337.788807:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829337.788810:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829338.492776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829338.492784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829338.492786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829338.492793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829338.492797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829338.492801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829338.492804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829338.492808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829338.492812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829339.516780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829339.516789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829339.516791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829339.516796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829339.516799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829339.516803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829339.516806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829339.516808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829339.516811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829340.028758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829340.028766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829340.028768:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829340.028771:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829340.028774:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829340.028781:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3487
00000400:00000200:1.0:1787829340.028788:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829340.028793:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829340.028795:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829340.028797:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028800:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.028801:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028803:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829340.028806:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829340.028808:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829340.028810:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829340.028812:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829340.028814:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829340.028815:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829340.028817:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028818:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.028820:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028821:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829340.028823:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829340.028826:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028827:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829340.028829:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829340.028832:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829340.028834:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.028835:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829340.028837:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829340.028845:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829340.028847:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829340.028850:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829340.028851:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.028853:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829340.028855:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829340.028858:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829340.028865:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829340.028871:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829340.028878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327334272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829340.028887:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829340.028890:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829340.028893:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327334272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829340.028913:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829340.028916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829340.028918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829340.028923:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829340.028924:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829340.028927:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829340.028928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3487
00000400:00000200:1.0:1787829340.028931:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829340.028933:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829340.028935:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829340.028937:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028939:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.028940:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028942:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829340.028943:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829340.028944:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829340.028946:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829340.028948:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829340.028949:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829340.028951:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829340.028954:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028955:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.028960:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028962:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829340.028966:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829340.028970:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829340.028972:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829340.028974:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829340.028979:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829340.028984:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.028988:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829340.028994:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829340.029000:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029003:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829340.029008:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829340.029011:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829340.029019:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829340.029022:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029028:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829340.029042:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829340.029048:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829340.029057:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829340.029064:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829340.029082:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327334400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829340.029092:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829340.029097:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829340.029104:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327334400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829340.029126:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829340.029132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829340.029138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829340.029141:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829340.029145:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829340.029151:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829340.029156:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3487
00000400:00000200:1.0:1787829340.029163:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829340.029168:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829340.029171:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829340.029178:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029181:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.029182:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029184:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829340.029189:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829340.029192:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829340.029194:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829340.029196:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829340.029199:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829340.029203:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829340.029208:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029211:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.029216:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029218:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829340.029222:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829340.029228:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029237:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829340.029241:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829340.029246:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829340.029255:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029259:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829340.029261:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829340.029265:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029268:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829340.029274:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829340.029277:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829340.029281:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829340.029283:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029285:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829340.029288:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829340.029291:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829340.029296:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829340.029304:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829340.029305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829340.029310:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00100000:1.0:1787829340.029312:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327334528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829340.029330:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829340.029334:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829340.029343:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327334528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829340.029354:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829340.029357:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829340.029360:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829340.029363:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829340.029365:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829340.029368:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829340.029373:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3487
00000400:00000200:1.0:1787829340.029379:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829340.029387:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829340.029389:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829340.029394:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029395:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.029397:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029398:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829340.029400:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829340.029402:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829340.029404:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829340.029406:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829340.029409:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829340.029413:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829340.029418:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029422:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829340.029426:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029430:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829340.029434:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829340.029438:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829340.029440:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829340.029446:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829340.029449:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829340.029452:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029453:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829340.029455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829340.029457:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029458:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829340.029461:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829340.029464:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829340.029467:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829340.029469:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829340.029472:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829340.029475:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829340.029477:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829340.029480:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829340.029484:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829340.029489:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327334656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829340.029495:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829340.029497:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829340.029503:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327334656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829340.029509:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829340.029515:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3497)
00000800:00000200:0.1:1787829340.029829:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829340.029831:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829340.030021:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829340.030023:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829340.030226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829340.030228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787829340.540759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829340.540768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829340.540770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829340.540774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829340.540777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829340.540781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829340.540784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829340.540787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829340.540790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829340.732725:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298160812, at 4298160832
00000100:00000040:3.0:1787829340.732728:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298160812, at 4298160832
00000100:00000010:2.0:1787829340.732749:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00000100:00000010:3.0:1787829340.732757:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000020:00000001:2.0:1787829340.860840:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829340.860846:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829340.860851:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829341.564778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829341.564785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829341.564787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829341.564792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829341.564795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829341.564800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829341.564802:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829341.564804:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829341.564807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829342.588742:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829342.588750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829342.588752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829342.588758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829342.588762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829342.588766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829342.588769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829342.588771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829342.588773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829343.612770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829343.612776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829343.612778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829343.612782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829343.612785:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829343.612787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829343.612789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829343.612791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829343.612793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829343.932875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829343.932881:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829343.932887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829344.636739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829344.636747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829344.636749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829344.636754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829344.636758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829344.636761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829344.636764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829344.636766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829344.636770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829345.148774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829345.148789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829345.148793:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829345.148797:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829345.148802:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829345.148810:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3492
00000400:00000200:1.0:1787829345.148820:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829345.148827:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829345.148830:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829345.148833:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.148835:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.148837:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.148839:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829345.148843:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829345.148845:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829345.148848:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829345.148851:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829345.148853:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829345.148854:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829345.148856:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.148857:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.148860:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.148861:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829345.148864:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829345.148867:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829345.148869:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829345.148872:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829345.148875:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829345.148877:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.148879:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829345.148881:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829345.148892:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829345.148895:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829345.148899:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829345.148901:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.148904:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829345.148906:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829345.148911:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829345.148919:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829345.148926:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829345.148936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327334784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829345.148945:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829345.148949:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829345.148953:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327334784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829345.148975:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829345.148978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829345.148980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829345.148981:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829345.148983:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829345.148985:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829345.148987:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3492
00000400:00000200:1.0:1787829345.148990:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829345.148992:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829345.149007:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829345.149010:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149012:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.149014:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149016:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829345.149018:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829345.149020:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829345.149021:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829345.149023:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829345.149025:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829345.149029:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829345.149032:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149037:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.149041:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149044:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829345.149047:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829345.149053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149058:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829345.149062:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829345.149069:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829345.149073:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149076:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829345.149083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829345.149087:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149091:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829345.149096:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829345.149101:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829345.149105:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829345.149108:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149110:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829345.149113:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829345.149117:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829345.149122:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829345.149128:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829345.149136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327334912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829345.149145:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829345.149147:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829345.149155:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327334912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829345.149166:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829345.149171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829345.149176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829345.149180:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829345.149184:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829345.149188:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829345.149192:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3492
00000400:00000200:1.0:1787829345.149197:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829345.149204:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829345.149207:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829345.149209:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149211:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.149215:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149218:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829345.149220:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829345.149222:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829345.149224:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829345.149227:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829345.149230:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829345.149235:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829345.149242:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149246:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.149250:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149253:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829345.149257:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829345.149264:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149269:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829345.149270:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829345.149273:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829345.149277:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149281:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829345.149285:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829345.149288:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149290:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829345.149294:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829345.149297:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829345.149298:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829345.149299:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149301:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829345.149303:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829345.149306:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829345.149310:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829345.149315:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829345.149322:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327335040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829345.149332:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829345.149335:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829345.149339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829345.149340:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327335040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829345.149346:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:1.0:1787829345.149349:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829345.149354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829345.149358:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829345.149360:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829345.149363:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829345.149365:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829345.149369:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3492
00000400:00000200:1.0:1787829345.149371:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829345.149373:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829345.149375:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829345.149379:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149383:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.149387:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149391:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829345.149404:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829345.149409:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829345.149414:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829345.149419:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829345.149425:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829345.149432:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829345.149441:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149446:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829345.149449:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149450:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829345.149453:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829345.149458:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829345.149461:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829345.149463:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829345.149468:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829345.149471:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149474:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829345.149476:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829345.149479:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149483:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829345.149487:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829345.149490:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829345.149496:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829345.149497:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829345.149498:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829345.149500:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829345.149501:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829345.149509:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829345.149515:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829345.149524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327335168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829345.149535:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829345.149538:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829345.149547:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327335168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829345.149566:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829345.149573:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3502)
00000800:00000200:0.1:1787829345.149853:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829345.149855:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829345.150063:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829345.150065:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829345.150246:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829345.150248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787829345.660769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829345.660776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829345.660778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829345.660783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829345.660786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829345.660789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829345.660791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829345.660793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829345.660796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829345.852760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298165932, at 4298165952
00000100:00000040:3.0:1787829345.852760:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298165932, at 4298165952
00000100:00000010:3.0:1787829345.852778:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:2.0:1787829345.852791:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000100:00000001:2.0:1787829346.684711:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829346.684718:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829346.684720:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829346.684725:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829346.684728:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829346.684731:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829346.684734:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829346.684736:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829346.684739:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829347.004849:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829347.004853:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829347.004856:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829347.708786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829347.708794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829347.708796:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829347.708801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829347.708805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829347.708809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829347.708812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829347.708814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829347.708817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829348.732777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829348.732788:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829348.732791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829348.732799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829348.732804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829348.732809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829348.732812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829348.732815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829348.732820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829349.756759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829349.756769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829349.756772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829349.756779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829349.756784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829349.756788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829349.756792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829349.756795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829349.756799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829350.076866:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829350.076870:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829350.076873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829350.268817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829350.268828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829350.268831:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829350.268835:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829350.268852:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829350.268861:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3497
00000400:00000200:3.0:1787829350.268870:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829350.268877:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829350.268880:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829350.268882:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.268885:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.268887:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.268890:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829350.268894:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829350.268896:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829350.268899:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829350.268902:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829350.268904:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829350.268905:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829350.268907:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.268909:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.268910:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.268911:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829350.268914:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829350.268917:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829350.268919:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829350.268922:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829350.268925:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829350.268928:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.268930:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829350.268932:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829350.268942:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829350.268945:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829350.268948:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829350.268950:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.268953:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829350.268956:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829350.268972:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829350.268983:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829350.268991:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829350.269014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327335296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829350.269024:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829350.269028:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829350.269033:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327335296/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829350.269053:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829350.269055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829350.269058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829350.269060:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829350.269061:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829350.269064:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829350.269065:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3497
00000400:00000200:3.0:1787829350.269068:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829350.269070:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829350.269072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829350.269075:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269077:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.269080:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269081:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829350.269083:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829350.269084:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829350.269086:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829350.269092:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829350.269097:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829350.269099:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829350.269104:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269107:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.269112:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269114:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829350.269119:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829350.269125:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269130:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829350.269135:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829350.269142:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829350.269148:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269154:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829350.269160:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829350.269165:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269169:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829350.269174:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829350.269177:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829350.269180:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829350.269181:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269183:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829350.269186:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829350.269189:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829350.269196:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829350.269203:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829350.269213:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327335424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829350.269226:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829350.269230:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829350.269236:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327335424/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829350.269252:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829350.269256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829350.269262:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829350.269265:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829350.269268:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829350.269272:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829350.269277:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3497
00000400:00000200:3.0:1787829350.269282:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829350.269286:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829350.269288:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829350.269292:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269297:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.269300:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269302:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829350.269305:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829350.269308:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829350.269310:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829350.269312:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829350.269313:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829350.269315:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829350.269318:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269320:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.269324:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269328:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829350.269331:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829350.269337:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269341:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829350.269344:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829350.269348:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829350.269353:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269357:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829350.269360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829350.269362:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269367:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829350.269373:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829350.269377:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829350.269381:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829350.269390:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269392:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829350.269393:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829350.269397:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829350.269404:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829350.269411:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829350.269414:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:3.0:1787829350.269419:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327335552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829350.269420:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000001:3.0:1787829350.269433:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829350.269436:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829350.269443:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327335552/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829350.269453:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829350.269457:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829350.269461:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829350.269462:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829350.269464:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829350.269473:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829350.269483:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3497
00000400:00000200:3.0:1787829350.269492:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829350.269501:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829350.269507:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829350.269513:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269519:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.269526:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269530:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829350.269534:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829350.269537:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829350.269538:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829350.269540:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829350.269542:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829350.269548:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829350.269553:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269557:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829350.269562:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269563:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829350.269566:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829350.269573:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829350.269576:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829350.269581:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829350.269585:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829350.269589:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269592:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829350.269596:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829350.269602:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269606:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829350.269614:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829350.269619:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829350.269625:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829350.269628:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829350.269633:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829350.269635:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829350.269637:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829350.269642:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829350.269648:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829350.269683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327335680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829350.269695:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829350.269700:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829350.269709:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327335680/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829350.269733:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829350.269739:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3508)
00000800:00000200:0.1:1787829350.269978:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829350.269997:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829350.270263:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829350.270266:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829350.270568:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829350.270569:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1787829350.780815:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829350.780823:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829350.780825:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829350.780830:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829350.780834:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829350.780837:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829350.780840:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829350.780843:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829350.780846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829350.972757:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298171052, at 4298171072
00000100:00000040:3.0:1787829350.972757:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298171052, at 4298171072
00000100:00000010:2.0:1787829350.972773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000010:3.0:1787829350.972775:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000100:00000001:2.0:1787829351.804752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829351.804757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829351.804759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829351.804763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829351.804766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829351.804768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829351.804770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829351.804772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829351.804774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829352.828776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829352.828787:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829352.828790:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829352.828798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829352.828803:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829352.828808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829352.828811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829352.828816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829352.828820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829353.148871:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829353.148874:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829353.148878:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829353.852738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829353.852743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829353.852745:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829353.852749:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829353.852751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829353.852754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829353.852756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829353.852757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829353.852760:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829354.876755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829354.876760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829354.876762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829354.876767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829354.876770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829354.876772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829354.876774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829354.876776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829354.876778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829355.388777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829355.388786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829355.388788:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829355.388791:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829355.388794:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829355.388800:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3503
00000400:00000200:1.0:1787829355.388808:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829355.388812:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829355.388815:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829355.388818:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.388820:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.388823:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.388825:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829355.388828:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829355.388829:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829355.388832:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829355.388834:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829355.388836:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829355.388837:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829355.388840:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.388841:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.388843:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.388844:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829355.388847:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829355.388849:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829355.388851:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829355.388854:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829355.388857:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829355.388859:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.388860:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829355.388862:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829355.388870:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829355.388872:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829355.388875:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829355.388877:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.388879:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829355.388881:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829355.388884:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829355.388891:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829355.388896:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829355.388904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327335808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829355.388911:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829355.388915:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829355.388919:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327335808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829355.388988:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829355.388991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829355.388993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829355.388994:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829355.388995:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829355.388997:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829355.388999:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3503
00000400:00000200:1.0:1787829355.389003:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829355.389006:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829355.389008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829355.389010:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389012:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.389014:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389016:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829355.389018:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829355.389019:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829355.389020:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829355.389022:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829355.389024:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829355.389029:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829355.389036:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389040:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.389044:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389048:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829355.389053:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829355.389059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389065:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829355.389068:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829355.389075:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829355.389078:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389079:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829355.389084:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829355.389089:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389093:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829355.389099:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829355.389104:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829355.389107:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829355.389110:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389113:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829355.389115:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829355.389117:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829355.389121:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829355.389127:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829355.389134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327335936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829355.389141:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829355.389145:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829355.389148:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327335936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829355.389159:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829355.389161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829355.389167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829355.389168:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829355.389171:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829355.389173:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829355.389177:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3503
00000400:00000200:1.0:1787829355.389181:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829355.389186:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829355.389191:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829355.389197:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389201:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.389204:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389207:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829355.389211:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829355.389213:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829355.389218:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829355.389220:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829355.389225:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829355.389231:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829355.389238:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389244:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.389252:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389258:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829355.389272:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829355.389282:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389291:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829355.389296:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829355.389300:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829355.389306:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389312:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.1:1787829355.389319:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00100000:1.0:1787829355.389324:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829355.389329:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389331:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829355.389336:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829355.389339:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829355.389345:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829355.389346:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389349:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829355.389353:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829355.389356:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829355.389364:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829355.389368:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829355.389375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327336064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829355.389383:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829355.389385:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829355.389391:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327336064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829355.389401:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829355.389404:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829355.389408:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829355.389411:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829355.389412:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829355.389414:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829355.389416:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3503
00000400:00000200:1.0:1787829355.389419:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829355.389423:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829355.389426:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829355.389431:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389437:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.389441:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389445:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829355.389457:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829355.389461:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829355.389465:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829355.389469:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829355.389475:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829355.389481:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829355.389489:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389495:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829355.389506:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389510:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829355.389514:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829355.389520:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829355.389524:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829355.389527:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829355.389530:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829355.389536:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389542:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829355.389548:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829355.389555:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389559:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829355.389567:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829355.389579:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829355.389585:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829355.389588:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829355.389591:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829355.389594:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829355.389597:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829355.389603:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829355.389607:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829355.389617:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327336192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829355.389634:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829355.389639:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829355.389703:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327336192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829355.389724:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829355.389735:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3513)
00000800:00000200:0.1:1787829355.389761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829355.389915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829355.390102:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829355.900769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829355.900779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829355.900782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829355.900789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829355.900794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829355.900800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829355.900804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829355.900807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829355.900811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829356.092717:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298176172, at 4298176192
00000100:00000040:3.0:1787829356.092727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298176172, at 4298176192
00000100:00000010:2.0:1787829356.092730:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000100:00000010:3.0:1787829356.092739:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000020:00000001:2.0:1787829356.220873:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829356.220879:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829356.220883:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829356.924785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829356.924791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829356.924792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829356.924797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829356.924799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829356.924802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829356.924805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829356.924807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829356.924809:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829357.948802:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829357.948807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829357.948809:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829357.948813:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829357.948816:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829357.948819:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829357.948821:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829357.948823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829357.948825:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1787829358.460956:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829358.460993:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829358.461001:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829358.461004:0:2334:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829358.461009:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829358.461012:0:2334:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829358.461015:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829358.461018:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:2.0:1787829358.461022:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829358.461026:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829358.461031:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefaa0
00000400:00000010:2.0:1787829358.461033:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefaa0.
00000100:00000001:2.0:1787829358.461037:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829358.461044:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e3480 x1874671327335808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461066:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829358.461071:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327335808/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461079:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829358.461082:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829358.461091:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829358.461093:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:2.0:1787829358.461096:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829358.461099:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829358.461102:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:2.0:1787829358.461104:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:2.0:1787829358.461107:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829358.461111:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e0780 x1874671327335936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461119:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829358.461123:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327335936/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461129:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829358.461131:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829358.461136:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829358.461139:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000200:2.0:1787829358.461142:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829358.461145:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829358.461147:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef5d8
00000400:00000010:2.0:1787829358.461149:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef5d8.
00000100:00000001:2.0:1787829358.461151:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829358.461156:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e3840 x1874671327336064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461164:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829358.461167:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327336064/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461175:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829358.461177:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829358.461181:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829358.461184:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000200:2.0:1787829358.461187:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829358.461189:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829358.461192:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef3b8
00000400:00000010:2.0:1787829358.461193:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef3b8.
00000100:00000001:2.0:1787829358.461195:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829358.461200:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e2940 x1874671327336192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461207:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829358.461210:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327336192/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829358.461217:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829358.461218:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829358.972776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829358.972780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829358.972782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829358.972786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829358.972789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829358.972793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829358.972795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829358.972797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829358.972799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829359.292911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829359.292917:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829359.292921:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829359.996738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829359.996746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829359.996748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829359.996755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829359.996759:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829359.996762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829359.996765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829359.996768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829359.996771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829360.508756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829360.508768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829360.508771:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829360.508775:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829360.508780:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829360.508788:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3508
00000400:00000200:1.0:1787829360.508797:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829360.508804:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829360.508807:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829360.508811:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.508814:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.508817:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.508819:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829360.508823:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829360.508825:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829360.508828:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829360.508831:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829360.508834:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829360.508837:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829360.508840:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.508842:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.508844:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.508846:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829360.508849:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829360.508853:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829360.508855:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829360.508858:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829360.508862:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829360.508865:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.508867:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829360.508870:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829360.508877:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829360.508881:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829360.508885:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829360.508900:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.508904:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829360.508907:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829360.508911:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829360.508920:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829360.508926:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829360.508936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327336320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829360.508946:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829360.508951:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829360.508956:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327336320/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829360.508984:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829360.508988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829360.508991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829360.508992:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829360.508995:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829360.508997:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829360.509000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3508
00000400:00000200:1.0:1787829360.509004:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829360.509008:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829360.509011:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829360.509014:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509018:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.509021:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509024:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829360.509027:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829360.509031:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829360.509035:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829360.509039:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829360.509044:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829360.509049:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829360.509058:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509064:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.509070:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509077:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829360.509083:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829360.509087:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509091:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829360.509096:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829360.509111:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829360.509117:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509122:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829360.509128:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829360.509133:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509136:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829360.509145:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829360.509150:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829360.509157:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829360.509160:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509166:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829360.509167:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829360.509170:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829360.509178:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829360.509182:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829360.509194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327336448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829360.509219:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829360.509226:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829360.509237:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327336448/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829360.509254:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829360.509259:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829360.509266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829360.509270:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829360.509272:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829360.509286:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829360.509294:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3508
00000400:00000200:1.0:1787829360.509318:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829360.509329:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829360.509335:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829360.509341:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509359:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.509366:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509369:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829360.509371:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829360.509373:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829360.509375:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829360.509378:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829360.509389:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829360.509395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829360.509396:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829360.509401:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:1.0:1787829360.509406:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509412:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.509418:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509425:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829360.509430:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829360.509439:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509447:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829360.509453:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829360.509459:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829360.509463:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509468:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829360.509472:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829360.509479:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509484:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829360.509491:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829360.509509:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829360.509516:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829360.509520:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509526:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829360.509529:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829360.509532:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829360.509540:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829360.509546:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829360.509556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327336576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829360.509569:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829360.509581:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829360.509587:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327336576/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829360.509607:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829360.509612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829360.509623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829360.509627:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829360.509630:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829360.509634:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829360.509644:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3508
00000400:00000200:1.0:1787829360.509689:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829360.509702:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829360.509709:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829360.509718:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509724:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.509731:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509737:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829360.509742:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829360.509746:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829360.509750:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829360.509754:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829360.509760:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829360.509767:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829360.509778:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509786:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829360.509794:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509801:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829360.509808:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829360.509818:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829360.509826:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829360.509833:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829360.509839:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829360.509847:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509853:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829360.509858:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829360.509863:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509868:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829360.509878:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829360.509886:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829360.509911:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829360.509915:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829360.509920:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829360.509925:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829360.509930:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829360.509938:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829360.509946:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829360.509957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327336704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829360.509984:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829360.509991:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829360.510003:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327336704/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829360.510025:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829360.510034:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3518)
00000800:00000200:0.1:1787829360.510151:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829360.510154:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829360.510384:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829360.510386:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829360.510729:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829360.510732:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787829361.020775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829361.020782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829361.020784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829361.020790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829361.020794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829361.020797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829361.020800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829361.020803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829361.020806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829361.212733:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298181292, at 4298181312
00000100:00000010:3.0:1787829361.212751:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00000040:2.0:1787829361.212772:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298181292, at 4298181312
00000100:00000010:2.0:1787829361.212788:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000001:2.0:1787829362.044772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829362.044780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829362.044782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829362.044788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829362.044792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829362.044796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829362.044798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829362.044801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829362.044804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829362.364871:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829362.364885:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829362.364888:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829363.068800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829363.068808:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829363.068810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829363.068815:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829363.068819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829363.068823:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829363.068825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829363.068828:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829363.068831:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829364.092733:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829364.092741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829364.092744:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829364.092749:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829364.092753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829364.092756:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829364.092759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829364.092762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829364.092764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829365.116782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829365.116790:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829365.116792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829365.116797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829365.116801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829365.116805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829365.116807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829365.116809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829365.116812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829365.436906:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829365.436911:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829365.436915:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829365.629199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829365.629208:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829365.629210:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829365.629213:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829365.629216:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829365.629222:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3513
00000400:00000200:1.0:1787829365.629231:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829365.629237:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829365.629240:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829365.629242:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629245:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629247:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629249:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829365.629252:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829365.629254:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829365.629255:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829365.629259:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829365.629261:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829365.629263:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829365.629265:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629266:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629268:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629269:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829365.629272:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829365.629275:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629277:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829365.629278:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829365.629281:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829365.629283:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629284:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829365.629286:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829365.629293:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829365.629296:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829365.629300:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829365.629302:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629305:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829365.629307:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829365.629312:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829365.629320:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829365.629326:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829365.629336:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327336832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829365.629348:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829365.629352:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829365.629356:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327336832/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829365.629371:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829365.629374:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829365.629376:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829365.629377:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829365.629378:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829365.629380:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829365.629383:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3513
00000400:00000200:1.0:1787829365.629387:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829365.629389:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829365.629392:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829365.629397:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629405:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629409:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629414:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829365.629418:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829365.629421:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829365.629424:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829365.629428:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829365.629433:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829365.629436:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829365.629443:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629449:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629456:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629463:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829365.629469:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829365.629476:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629486:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829365.629489:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829365.629497:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829365.629503:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629506:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829365.629511:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829365.629515:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629518:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829365.629522:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829365.629526:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829365.629528:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829365.629532:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629535:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829365.629537:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829365.629540:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829365.629545:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829365.629551:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829365.629560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327336960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829365.629570:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829365.629573:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829365.629578:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327336960/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829365.629587:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829365.629590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829365.629596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829365.629600:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829365.629602:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829365.629607:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829365.629613:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3513
00000400:00000200:1.0:1787829365.629630:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829365.629638:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829365.629641:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829365.629647:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629650:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629654:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629682:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829365.629686:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829365.629687:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829365.629691:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829365.629693:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829365.629698:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829365.629703:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829365.629704:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829365.629710:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1787829365.629711:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629715:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629720:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629724:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829365.629727:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829365.629731:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629737:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829365.629740:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829365.629743:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829365.629747:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629749:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829365.629752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829365.629756:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629761:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829365.629766:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829365.629770:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829365.629775:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829365.629777:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.629780:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829365.629782:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829365.629785:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829365.629789:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829365.629795:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829365.629803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327337088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829365.629812:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829365.629816:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829365.629822:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327337088/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829365.629834:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829365.629840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829365.629847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829365.629863:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829365.629867:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829365.629870:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829365.629874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3513
00000400:00000200:1.0:1787829365.629884:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829365.629892:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829365.629899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829365.629908:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629913:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629919:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629926:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829365.629932:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829365.629935:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829365.629938:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829365.629941:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829365.629946:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829365.629951:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829365.629957:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629961:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829365.629964:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629969:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829365.629973:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829365.629979:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829365.629984:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829365.629988:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829365.629991:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829365.629996:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.630000:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829365.630004:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829365.630010:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.630015:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829365.630020:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829365.630026:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829365.630031:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829365.630034:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829365.630038:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829365.630041:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829365.630045:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829365.630051:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829365.630057:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829365.630067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327337216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829365.630078:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829365.630082:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829365.630091:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327337216/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829365.630103:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829365.630109:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3523)
00000800:00000200:0.1:1787829365.630251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829365.630254:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829365.630485:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829365.630488:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829365.630689:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829365.630691:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829366.140738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829366.140745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829366.140747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829366.140752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829366.140755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829366.140759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829366.140762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829366.140764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829366.140767:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829366.332780:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298186413, at 4298186432
00000100:00000040:3.0:1787829366.332780:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298186413, at 4298186432
00000100:00000010:3.0:1787829366.332799:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000100:00000010:2.0:1787829366.332802:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000001:2.0:1787829367.164775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829367.164784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829367.164786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829367.164792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829367.164796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829367.164800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829367.164803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829367.164805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829367.164807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829368.188783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829368.188791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829368.188792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829368.188798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829368.188801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829368.188805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829368.188808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829368.188811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829368.188813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829368.508911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829368.508914:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829368.508917:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829369.212749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829369.212757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829369.212760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829369.212764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829369.212768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829369.212772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829369.212774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829369.212776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829369.212779:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829370.236724:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829370.236730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829370.236732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829370.236736:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829370.236739:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829370.236741:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829370.236743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829370.236745:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829370.236747:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829370.748793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829370.748805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829370.748808:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829370.748811:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829370.748827:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829370.748835:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3518
00000400:00000200:3.0:1787829370.748845:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829370.748851:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829370.748855:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829370.748859:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.748863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.748865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.748868:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829370.748873:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829370.748875:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829370.748878:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829370.748881:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829370.748884:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829370.748887:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829370.748890:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.748893:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.748895:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.748897:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829370.748900:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829370.748905:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829370.748908:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829370.748911:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829370.748914:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829370.748917:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.748920:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829370.748923:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829370.748931:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829370.748935:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829370.748939:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829370.748942:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.748945:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829370.748948:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829370.748952:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829370.748960:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829370.748966:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829370.748975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327337344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829370.748986:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829370.748991:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829370.748996:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327337344/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829370.749019:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829370.749023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829370.749026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829370.749028:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829370.749030:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829370.749033:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829370.749036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3518
00000400:00000200:3.0:1787829370.749041:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829370.749045:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829370.749047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829370.749052:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749054:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.749057:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749060:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829370.749063:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829370.749065:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829370.749067:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829370.749071:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829370.749077:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829370.749081:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829370.749092:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749099:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.749106:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749118:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829370.749128:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829370.749140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749145:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829370.749152:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829370.749162:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829370.749167:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.749173:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829370.749181:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829370.749188:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.749195:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829370.749203:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829370.749208:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829370.749213:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829370.749219:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.749229:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829370.749232:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829370.749235:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829370.749241:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829370.749247:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829370.749262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327337472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829370.749280:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829370.749285:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829370.749299:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327337472/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829370.749328:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829370.749334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829370.749342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829370.749344:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829370.749348:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829370.749360:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829370.749365:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3518
00000400:00000200:3.0:1787829370.749375:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829370.749383:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829370.749388:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829370.749402:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749408:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.749419:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749427:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829370.749436:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829370.749439:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829370.749446:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829370.749452:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829370.749463:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829370.749468:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829370.749478:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749488:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.749502:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749512:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829370.749518:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829370.749528:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749540:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829370.749552:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787829370.749557:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:3.0:1787829370.749561:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.1:1787829370.749561:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
02000000:00000001:3.0:1787829370.749576:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.749583:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829370.749589:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829370.749602:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.749607:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829370.749617:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829370.749624:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829370.749634:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829370.749638:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.749642:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829370.749646:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829370.749689:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829370.749696:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829370.749705:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829370.749716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327337600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829370.749729:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829370.749734:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829370.749745:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327337600/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829370.749766:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829370.749771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829370.749776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829370.749782:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829370.749785:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829370.749788:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829370.749796:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3518
00000400:00000200:3.0:1787829370.749806:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829370.749830:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829370.749836:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829370.749845:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749852:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.749861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749869:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829370.749876:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829370.749881:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829370.749885:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829370.749891:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829370.749900:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829370.749910:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829370.749923:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749931:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829370.749938:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749945:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829370.749956:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829370.749967:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829370.749972:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829370.749976:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829370.749982:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829370.749996:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.750004:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829370.750009:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829370.750016:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.750020:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829370.750026:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829370.750034:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829370.750044:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829370.750047:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829370.750054:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829370.750058:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829370.750065:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829370.750074:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829370.750084:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829370.750099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327337728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829370.750129:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829370.750139:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829370.750152:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327337728/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829370.750191:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829370.750209:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3528)
00000800:00000200:0.1:1787829370.750380:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829370.750382:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829370.750577:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829370.750578:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829370.750782:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829370.750784:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829371.260744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829371.260755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829371.260758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829371.260765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829371.260770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829371.260775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829371.260778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829371.260781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829371.260785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829371.452716:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298191533, at 4298191552
00000100:00000040:2.0:1787829371.452725:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298191533, at 4298191552
00000100:00000010:3.0:1787829371.452730:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000010:2.0:1787829371.452740:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000020:00000001:2.0:1787829371.580915:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829371.580920:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829371.580924:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829372.284796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829372.284804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829372.284817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829372.284823:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829372.284827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829372.284831:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829372.284834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829372.284836:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829372.284838:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829373.308765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829373.308770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829373.308772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829373.308777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829373.308779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829373.308782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829373.308784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829373.308786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829373.308789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829374.652806:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829374.652810:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829374.652812:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829375.868756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829375.868768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829375.868772:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829375.868792:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829375.868798:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829375.868805:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3523
00000400:00000200:3.0:1787829375.868815:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829375.868821:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829375.868824:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829375.868827:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.868830:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.868832:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.868835:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829375.868838:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829375.868840:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829375.868843:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829375.868846:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829375.868848:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829375.868850:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829375.868852:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.868853:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.868854:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.868856:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829375.868859:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829375.868862:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829375.868864:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829375.868866:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829375.868870:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829375.868872:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.868874:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829375.868875:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829375.868886:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829375.868889:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829375.868893:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829375.868895:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.868898:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829375.868901:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829375.868905:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829375.868914:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829375.868922:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829375.868934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327337856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829375.868946:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829375.868953:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829375.868960:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327337856/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829375.868983:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829375.868986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829375.868989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829375.868990:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829375.868991:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829375.868994:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829375.868996:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3523
00000400:00000200:3.0:1787829375.868999:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829375.869001:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829375.869003:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829375.869006:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869008:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.869011:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869013:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829375.869016:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829375.869016:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829375.869018:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829375.869020:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829375.869023:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829375.869025:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829375.869029:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869032:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.869037:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869039:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829375.869043:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829375.869049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869055:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829375.869060:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829375.869067:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829375.869071:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869074:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829375.869081:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829375.869085:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869089:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829375.869094:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829375.869098:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829375.869102:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829375.869105:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869107:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829375.869108:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829375.869110:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829375.869121:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829375.869130:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829375.869142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327337984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829375.869159:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829375.869165:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829375.869177:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327337984/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829375.869199:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829375.869206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829375.869213:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829375.869215:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829375.869217:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829375.869222:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829375.869227:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3523
00000400:00000200:3.0:1787829375.869234:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829375.869241:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829375.869244:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829375.869249:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869252:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.869257:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869262:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829375.869264:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829375.869266:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829375.869268:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829375.869272:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829375.869276:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829375.869281:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829375.869285:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869288:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.869293:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869298:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829375.869303:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829375.869309:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869311:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829375.869312:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829375.869315:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829375.869317:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869320:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829375.869323:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829375.869328:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869332:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829375.869337:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829375.869340:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829375.869346:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829375.869348:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869353:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829375.869354:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829375.869355:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829375.869360:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829375.869365:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829375.869371:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327338112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829375.869382:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829375.869387:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829375.869393:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327338112/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829375.869395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829375.869403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:3.0:1787829375.869405:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829375.869409:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829375.869416:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829375.869420:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829375.869422:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829375.869426:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829375.869430:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3523
00000400:00000200:3.0:1787829375.869435:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829375.869442:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829375.869445:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829375.869448:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869452:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.869457:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869459:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829375.869463:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829375.869465:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829375.869468:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829375.869471:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829375.869476:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829375.869480:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829375.869486:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869489:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829375.869494:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869499:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829375.869503:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829375.869509:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829375.869512:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829375.869514:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829375.869518:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829375.869521:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869523:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829375.869525:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829375.869528:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869532:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829375.869535:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829375.869538:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829375.869542:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829375.869546:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829375.869551:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829375.869555:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829375.869560:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829375.869567:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829375.869574:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829375.869582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327338240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829375.869598:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829375.869604:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829375.869615:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327338240/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829375.869641:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829375.869646:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3533)
00000800:00000200:0.1:1787829375.869969:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829375.869970:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829375.870266:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829375.870267:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829375.870464:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829375.870466:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000040:3.0:1787829376.572716:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298196652, at 4298196672
00000100:00000040:2.0:1787829376.572717:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298196652, at 4298196672
00000100:00000010:2.0:1787829376.572739:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000100:00000010:3.0:1787829376.572742:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000020:00000001:2.0:1787829377.724822:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829377.724825:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829377.724828:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829380.796837:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829380.796840:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829380.796844:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829380.988821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829380.988832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829380.988835:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829380.988838:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829380.988850:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829380.988856:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3528
00000400:00000200:2.0:1787829380.988863:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829380.988869:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829380.988871:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829380.988874:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.988876:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.988878:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.988879:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829380.988882:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829380.988884:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829380.988885:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829380.988888:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829380.988890:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829380.988891:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829380.988894:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.988896:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.988897:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.988898:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829380.988900:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829380.988903:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829380.988904:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829380.988906:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829380.988909:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829380.988911:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.988912:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829380.988914:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829380.988922:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829380.988925:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829380.988928:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829380.988929:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.988931:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829380.988933:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829380.988936:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829380.988943:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829380.988950:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829380.988959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327338368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829380.988971:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829380.988977:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829380.988982:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327338368/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829380.989006:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829380.989009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829380.989011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829380.989012:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829380.989014:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829380.989016:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829380.989018:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3528
00000400:00000200:2.0:1787829380.989021:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829380.989024:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829380.989025:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829380.989027:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989029:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.989031:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989032:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829380.989034:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829380.989035:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829380.989037:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829380.989038:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829380.989040:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829380.989042:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829380.989044:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989045:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.989047:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989052:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829380.989055:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829380.989059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989063:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829380.989065:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829380.989071:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829380.989075:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989079:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829380.989084:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829380.989088:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989092:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829380.989096:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829380.989101:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829380.989106:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829380.989109:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989111:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829380.989113:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829380.989116:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829380.989121:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829380.989126:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829380.989133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327338496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829380.989146:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829380.989148:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829380.989153:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327338496/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829380.989162:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829380.989166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829380.989168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829380.989169:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829380.989171:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829380.989174:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829380.989177:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3528
00000400:00000200:2.0:1787829380.989180:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829380.989182:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829380.989184:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829380.989190:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989195:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.989199:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989202:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829380.989204:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829380.989207:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829380.989211:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829380.989214:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829380.989220:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829380.989226:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829380.989234:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989239:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.989254:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989260:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829380.989265:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829380.989272:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989278:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829380.989283:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829380.989289:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829380.989296:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989301:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829380.989305:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829380.989310:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989315:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829380.989321:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829380.989326:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829380.989332:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829380.989333:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989335:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829380.989337:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829380.989340:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829380.989345:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829380.989351:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829380.989359:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327338624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829380.989361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829380.989367:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1787829380.989371:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829380.989375:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829380.989380:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327338624/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829380.989390:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829380.989393:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829380.989395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829380.989399:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829380.989401:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829380.989405:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829380.989409:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3528
00000400:00000200:2.0:1787829380.989415:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829380.989418:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829380.989422:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829380.989425:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989428:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.989430:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989435:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829380.989436:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829380.989438:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829380.989440:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829380.989443:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829380.989447:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829380.989451:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829380.989457:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989461:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829380.989465:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989469:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829380.989474:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829380.989479:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829380.989485:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829380.989491:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829380.989495:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829380.989497:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989499:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829380.989502:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829380.989504:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989507:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829380.989513:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787829380.989517:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829380.989519:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829380.989520:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829380.989522:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829380.989524:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829380.989526:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829380.989531:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829380.989536:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829380.989548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327338752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829380.989563:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829380.989566:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829380.989571:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327338752/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829380.989576:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829380.989581:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3538)
00000800:00000200:0.1:1787829380.989887:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829380.989888:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829380.990085:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829380.990087:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829380.990304:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829380.990306:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1787829381.692773:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298201772, at 4298201792
00000100:00000040:3.0:1787829381.692773:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298201772, at 4298201792
00000100:00000010:3.0:1787829381.692789:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000100:00000010:2.0:1787829381.692792:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000020:00000001:2.0:1787829383.868874:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829383.868877:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829383.868880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829386.108827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829386.108839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829386.108842:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829386.108846:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829386.108851:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829386.108858:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3533
00000400:00000200:3.0:1787829386.108868:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829386.108874:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829386.108878:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829386.108880:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.108883:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.108885:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.108887:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829386.108891:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829386.108893:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829386.108896:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829386.108899:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829386.108901:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829386.108902:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829386.108904:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.108905:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.108907:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.108909:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829386.108911:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829386.108914:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829386.108916:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829386.108918:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829386.108922:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829386.108924:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.108927:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829386.108930:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829386.108940:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829386.108943:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829386.108947:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829386.108949:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.108952:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829386.108955:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829386.108959:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829386.108968:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829386.108983:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829386.108993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327338880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829386.109003:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829386.109007:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829386.109012:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327338880/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829386.109029:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829386.109032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829386.109035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829386.109036:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829386.109037:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829386.109039:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829386.109041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3533
00000400:00000200:3.0:1787829386.109043:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829386.109045:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829386.109047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829386.109049:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109051:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.109053:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109055:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829386.109058:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829386.109059:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829386.109060:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829386.109062:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829386.109064:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829386.109066:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829386.109069:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109071:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.109073:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109074:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829386.109078:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829386.109081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109084:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829386.109086:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829386.109092:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829386.109097:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109100:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829386.109106:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829386.109110:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109112:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829386.109115:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829386.109119:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829386.109123:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829386.109126:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109130:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829386.109133:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829386.109135:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829386.109139:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829386.109144:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829386.109152:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327339008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829386.109161:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829386.109165:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829386.109170:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327339008/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829386.109181:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829386.109185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829386.109189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829386.109190:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829386.109192:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829386.109197:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829386.109200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3533
00000400:00000200:3.0:1787829386.109203:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829386.109208:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829386.109210:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829386.109215:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109218:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.109222:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109227:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829386.109231:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829386.109239:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829386.109242:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829386.109245:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829386.109247:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829386.109249:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829386.109254:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109259:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.109263:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109266:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829386.109269:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829386.109272:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109276:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829386.109277:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829386.109281:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829386.109285:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109287:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829386.109291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829386.109296:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109301:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829386.109306:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829386.109311:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829386.109314:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829386.109315:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109318:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829386.109320:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829386.109323:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829386.109326:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829386.109329:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829386.109336:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327339136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829386.109348:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829386.109352:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829386.109359:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327339136/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829386.109380:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829386.109385:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829386.109389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829386.109393:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829386.109396:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829386.109402:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829386.109407:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3533
00000400:00000200:3.0:1787829386.109416:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829386.109423:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829386.109426:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829386.109432:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109434:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.109440:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109445:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829386.109448:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829386.109449:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829386.109451:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829386.109453:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829386.109458:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829386.109460:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829386.109464:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829386.109466:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829386.109467:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829386.109471:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829386.109475:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000001:3.0:1787829386.109476:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829386.109481:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829386.109488:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829386.109495:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829386.109499:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829386.109504:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829386.109507:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109513:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829386.109517:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829386.109522:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109526:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829386.109531:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829386.109537:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829386.109541:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829386.109544:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829386.109547:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829386.109550:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829386.109553:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829386.109559:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829386.109564:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829386.109572:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327339264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829386.109595:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829386.109599:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829386.109604:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327339264/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829386.109610:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829386.109616:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3543)
00000800:00000200:0.1:1787829386.109961:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829386.109963:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829386.110238:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829386.110241:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829386.110489:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829386.110492:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00000040:2.0:1787829386.812777:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298206892, at 4298206912
00000100:00000040:3.0:1787829386.812777:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298206892, at 4298206912
00000100:00000010:3.0:1787829386.812791:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000100:00000010:2.0:1787829386.812794:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000020:00000001:2.0:1787829386.940872:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829386.940877:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829386.940882:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829390.012888:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829390.012891:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829390.012894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829391.228787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829391.228795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829391.228797:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829391.228800:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829391.228804:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829391.228809:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3538
00000400:00000200:2.0:1787829391.228816:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829391.228821:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829391.228824:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829391.228827:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228829:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.228831:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228832:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829391.228835:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829391.228837:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829391.228838:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829391.228841:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829391.228843:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829391.228844:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829391.228846:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228847:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.228849:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228850:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829391.228852:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829391.228855:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228857:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829391.228859:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829391.228862:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829391.228864:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.228866:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829391.228868:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829391.228877:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829391.228879:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829391.228882:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829391.228883:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.228886:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829391.228887:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829391.228891:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829391.228897:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787829391.228901:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829391.228908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327339392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829391.228916:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829391.228919:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829391.228922:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327339392/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829391.228942:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829391.228945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829391.228949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829391.228951:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829391.228952:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829391.228955:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829391.228958:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3538
00000400:00000200:2.0:1787829391.228961:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829391.228965:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829391.228968:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829391.228971:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228975:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.228979:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.228982:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829391.228985:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829391.228986:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829391.228988:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829391.228991:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829391.228994:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829391.229002:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829391.229011:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229014:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.229021:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229025:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829391.229027:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829391.229030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229033:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829391.229038:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829391.229049:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829391.229053:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229065:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829391.229070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829391.229077:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229081:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829391.229089:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787829391.229100:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829391.229110:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829391.229113:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229116:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829391.229122:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829391.229128:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829391.229141:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829391.229152:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829391.229166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327339520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829391.229177:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829391.229190:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829391.229203:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327339520/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829391.229222:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829391.229226:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829391.229239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829391.229243:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829391.229248:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829391.229261:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829391.229273:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3538
00000400:00000200:2.0:1787829391.229288:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829391.229294:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829391.229305:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829391.229312:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229320:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.229324:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229326:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829391.229328:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829391.229328:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829391.229330:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829391.229335:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829391.229339:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829391.229341:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829391.229346:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229355:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.229362:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229366:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829391.229374:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829391.229389:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229396:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829391.229407:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829391.229415:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829391.229425:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229428:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829391.229430:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829391.229436:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229445:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829391.229458:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787829391.229468:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829391.229470:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829391.229471:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229472:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829391.229475:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829391.229478:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829391.229482:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829391.229489:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829391.229502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327339648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829391.229511:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829391.229515:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829391.229522:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327339648/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829391.229531:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829391.229545:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829391.229546:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00080000:2.0:1787829391.229548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829391.229556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829391.229558:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829391.229561:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829391.229567:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829391.229577:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3538
00000400:00000200:2.0:1787829391.229587:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829391.229593:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829391.229602:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829391.229613:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229620:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.229631:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229638:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829391.229647:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829391.229654:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829391.229714:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829391.229717:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829391.229726:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829391.229731:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829391.229738:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229741:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829391.229744:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229750:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829391.229757:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829391.229764:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829391.229770:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829391.229772:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829391.229777:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829391.229780:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229783:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829391.229785:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829391.229789:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229791:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829391.229795:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787829391.229802:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829391.229813:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829391.229817:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829391.229821:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829391.229822:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829391.229823:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829391.229828:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829391.229833:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829391.229840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327339776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829391.229856:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829391.229862:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829391.229871:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327339776/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829391.229949:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829391.229963:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3549)
00000800:00000200:0.1:1787829391.230203:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829391.230206:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829391.230425:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829391.230427:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829391.230652:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829391.230654:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787829391.932774:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298212013, at 4298212032
00000100:00000040:3.0:1787829391.932774:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298212013, at 4298212032
00000100:00000010:3.0:1787829391.932789:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000100:00000010:2.0:1787829391.932791:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000020:00000001:2.0:1787829393.084901:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829393.084907:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829393.084911:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829396.156809:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829396.156814:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829396.156818:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829396.349018:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829396.349024:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829396.349028:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829396.349038:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349053:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829396.349056:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.349060:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829396.349063:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829396.349068:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829396.349072:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327339904:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829396.349078:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829396.349084:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829396.349088:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.349095:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829396.349100:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829396.349104:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829396.349113:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829396.349118:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829396.349128:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829396.349138:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327339904, portal 25
00000100:00000001:1.0:1787829396.349143:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829396.349145:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829396.349154:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349169:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829396.349175:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829396.349178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829396.349183:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327339904, offset 0
00000400:00000200:1.0:1787829396.349192:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829396.349203:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.349221:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829396.349229:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829396.349236:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829396.349243:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829396.349247:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829396.349255:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40427:40427:256:4294967295] 192.168.203.116@tcp LPNI seq info [40427:40427:8:4294967295]
00000400:00000200:1.0:1787829396.349263:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.349279:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829396.349292:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.349298:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829396.349311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829396.349316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829396.349323:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829396.349328:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829396.349337:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.349341:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829396.349348:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829396.349356:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829396.349362:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829396.349366:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829396.349372:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829396.349378:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349395:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.349402:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349415:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.349419:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829396.349427:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829396.349432:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829396.349434:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298217132
00000100:00000010:1.0:1787829396.349440:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e638a0.
00000100:00000001:1.0:1787829396.349450:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829396.349461:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298217132
00000100:00000010:1.0:1787829396.349469:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63360.
00000100:00000001:1.0:1787829396.349478:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829396.349497:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829396.349504:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.349508:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829396.349515:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829396/real 1787829396]  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829396.349532:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829396.349535:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829396.349537:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829396.349543:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829396.349547:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829396.349550:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829396.349555:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829396.349567:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349574:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829396.349577:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349579:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.349583:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829396.349594:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349614:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829396.349622:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349634:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829396.349651:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349653:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.349682:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349685:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829396.349686:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829396.349689:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829396.349697:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829396.349703:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829396.349709:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829396.349719:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327339904:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829396.349724:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349726:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.349728:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327339904/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349738:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829396.349741:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829396.349744:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829396.349746:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829396.349748:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829396.349749:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829396.349751:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829396.349753:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829396.349757:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829396.349759:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829396.349760:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829396.349762:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829396.349763:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349765:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349771:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829396.349773:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.349775:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829396.349776:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829396.349777:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829396.349779:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349784:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829396.349785:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.349787:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829396.349789:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829396.349790:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829396.349792:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327340032:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829396.349794:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829396.349795:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829396.349796:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.349798:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829396.349800:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829396.349802:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829396.349805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829396.349807:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829396.349810:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829396.349813:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327340032, portal 10
00000100:00000001:1.0:1787829396.349814:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829396.349815:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829396.349821:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349826:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829396.349828:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829396.349829:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829396.349831:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327340032, offset 0
00000400:00000200:1.0:1787829396.349834:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829396.349837:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.349839:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829396.349842:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829396.349843:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829396.349845:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829396.349847:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829396.349849:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40428:40428:256:4294967295] 192.168.203.116@tcp LPNI seq info [40428:40428:8:4294967295]
00000400:00000200:1.0:1787829396.349852:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.349857:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829396.349860:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.349863:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829396.349865:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829396.349868:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829396.349871:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829396.349873:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829396.349876:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.349877:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829396.349879:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829396.349882:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829396.349884:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829396.349886:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829396.349888:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829396.349890:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349896:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.349898:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349903:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.349904:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829396.349906:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829396.349909:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829396.349911:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829396.349912:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298216459
00000100:00000001:1.0:1787829396.349914:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298217132
00000100:00000001:1.0:1787829396.349916:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298216459
00000100:00000001:1.0:1787829396.349917:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298217132
00000100:00000001:1.0:1787829396.349919:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829396.349920:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.349922:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829396.349924:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829396/real 1787829396]  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829396.349928:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829396.349929:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829396.349930:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829396.349932:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829396.349933:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829396.349934:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829396.349936:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829396.349941:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349949:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829396.349951:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349952:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.349953:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829396.349955:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349959:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829396.349961:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349965:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829396.349967:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349968:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.349969:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.349970:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829396.349971:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829396.349973:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829396.349974:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829396.349976:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829396.349978:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829396.349983:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327340032:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829396.349985:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829396.349987:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.349994:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829396.349996:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.349999:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829396.350000:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829396.350003:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829396.350006:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327340160:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829396.350010:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829396.350012:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829396.350014:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.350016:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829396.350019:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829396.350021:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829396.350025:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829396.350028:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829396.350032:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829396.350036:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327340160, portal 4
00000100:00000001:1.0:1787829396.350038:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829396.350040:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829396.350046:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350053:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829396.350056:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829396.350059:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829396.350062:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327340160, offset 0
00000400:00000200:1.0:1787829396.350066:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829396.350071:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.350075:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829396.350078:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829396.350081:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829396.350084:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829396.350086:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829396.350089:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40429:40429:256:4294967295] 192.168.203.116@tcp LPNI seq info [40429:40429:8:4294967295]
00000400:00000200:1.0:1787829396.350094:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.350101:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829396.350107:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.350111:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829396.350115:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829396.350118:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829396.350123:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829396.350126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829396.350131:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.350133:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829396.350136:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829396.350140:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829396.350144:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829396.350146:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829396.350149:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829396.350152:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350161:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.350164:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350171:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.350173:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829396.350177:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829396.350180:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829396.350184:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829396.350187:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298216466
00000100:00000001:1.0:1787829396.350189:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298217132
00000100:00000001:1.0:1787829396.350191:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298216466
00000100:00000001:1.0:1787829396.350193:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298217132
00000100:00000001:1.0:1787829396.350195:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829396.350196:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.350197:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829396.350200:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829396/real 1787829396]  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829396.350207:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829396.350208:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829396.350210:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829396.350212:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829396.350213:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829396.350214:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829396.350216:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829396.350222:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350226:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829396.350227:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350228:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.350229:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829396.350231:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350236:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829396.350238:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350243:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829396.350245:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350246:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.350246:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350247:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829396.350248:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829396.350250:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829396.350252:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829396.350253:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829396.350257:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829396.350262:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327340160:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829396.350264:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829396.350266:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350270:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829396.350271:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.350273:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829396.350274:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829396.350275:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829396.350277:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327340288:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829396.350279:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829396.350280:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829396.350281:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829396.350282:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829396.350285:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829396.350286:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829396.350288:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829396.350290:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829396.350292:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829396.350294:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327340288, portal 4
00000100:00000001:1.0:1787829396.350296:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829396.350297:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829396.350301:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350306:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829396.350307:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829396.350309:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829396.350311:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327340288, offset 0
00000400:00000200:1.0:1787829396.350314:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829396.350317:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.350319:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829396.350321:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829396.350323:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829396.350325:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829396.350327:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829396.350329:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40430:40430:256:4294967295] 192.168.203.116@tcp LPNI seq info [40430:40430:8:4294967295]
00000400:00000200:1.0:1787829396.350332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829396.350336:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829396.350339:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.350341:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829396.350344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829396.350346:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829396.350349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829396.350351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829396.350353:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829396.350355:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829396.350357:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829396.350359:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829396.350361:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829396.350362:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829396.350364:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829396.350366:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350371:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.350373:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350378:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.350379:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829396.350381:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829396.350383:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829396.350385:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829396.350387:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298216471
00000100:00000001:1.0:1787829396.350389:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298217132
00000100:00000001:1.0:1787829396.350390:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298216471
00000100:00000001:1.0:1787829396.350392:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298217132
00000100:00000001:1.0:1787829396.350393:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829396.350394:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.350395:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829396.350397:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829396/real 1787829396]  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829396.350402:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829396.350403:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829396.350404:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829396.350405:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829396.350407:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829396.350408:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829396.350410:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829396.350415:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350419:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829396.350420:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350421:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.350421:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829396.350423:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350427:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829396.350430:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350434:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829396.350436:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350437:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829396.350438:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350439:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829396.350440:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829396.350442:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829396.350444:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829396.350445:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829396.350448:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829396.350452:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327340288:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829396.350455:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350457:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.350460:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327340032/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350467:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829396.350469:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829396.350470:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829396.350473:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829396.350474:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829396.350476:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829396.350478:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829396.350481:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829396.350483:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829396.350485:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829396.350486:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829396.350489:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829396.350490:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350493:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.350497:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327340160/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350510:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829396.350512:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829396.350513:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829396.350516:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829396.350518:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829396.350519:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829396.350521:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829396.350523:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:1.0:1787829396.350525:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829396.350527:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829396.350528:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829396.350531:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829396.350532:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350534:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829396.350538:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327340288/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829396.350545:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829396.350547:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829396.350548:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829396.350550:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829396.350551:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829396.350552:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829396.350553:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829396.350554:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829396.350556:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829396.350557:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829396.350558:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829396.350559:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829396.350560:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350561:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829396.350564:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829396.350565:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829396.350566:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829396.350567:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829397.052716:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298217132, at 4298217152
00000100:00000040:3.0:1787829397.052728:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298217132, at 4298217152
00000100:00000010:2.0:1787829397.052738:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e638a0.
00000100:00000010:3.0:1787829397.052739:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63360.
00000020:00000001:2.0:1787829399.228836:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829399.228842:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829399.228845:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829401.468767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829401.468777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829401.468780:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829401.468784:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829401.468788:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829401.468796:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3549
00000400:00000200:1.0:1787829401.468805:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829401.468812:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829401.468815:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829401.468818:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468821:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.468823:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468825:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829401.468828:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829401.468830:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829401.468833:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829401.468836:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829401.468838:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829401.468840:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829401.468842:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468843:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.468844:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468846:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829401.468848:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829401.468851:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468853:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829401.468855:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829401.468859:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829401.468861:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.468863:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829401.468865:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829401.468875:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829401.468878:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829401.468882:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829401.468884:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.468887:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829401.468890:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829401.468895:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829401.468903:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829401.468909:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829401.468919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327340416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829401.468934:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829401.468937:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829401.468941:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327340416/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829401.468964:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829401.468967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829401.468969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829401.468970:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829401.468972:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829401.468974:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829401.468976:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3549
00000400:00000200:1.0:1787829401.468978:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829401.468980:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829401.468982:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829401.468984:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468986:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.468987:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.468988:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829401.468989:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829401.468990:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829401.468991:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829401.468994:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829401.468996:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829401.468998:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829401.469001:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469002:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.469005:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469007:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829401.469010:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829401.469017:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469023:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829401.469028:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829401.469036:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829401.469041:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469046:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829401.469052:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829401.469059:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469063:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829401.469070:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829401.469075:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829401.469079:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829401.469082:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469084:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829401.469088:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829401.469090:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829401.469093:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829401.469098:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829401.469108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327340544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829401.469128:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829401.469133:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829401.469137:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327340544/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829401.469157:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829401.469161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829401.469166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829401.469169:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829401.469172:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829401.469178:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829401.469183:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3549
00000400:00000200:1.0:1787829401.469191:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829401.469196:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829401.469199:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829401.469204:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469206:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.469211:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469214:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829401.469215:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829401.469216:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829401.469217:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829401.469219:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829401.469220:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829401.469226:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829401.469231:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469233:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.469234:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469235:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829401.469240:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829401.469246:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469248:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829401.469252:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829401.469255:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829401.469258:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469260:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829401.469263:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829401.469266:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469267:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829401.469271:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829401.469275:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829401.469278:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829401.469281:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469283:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829401.469286:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829401.469289:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829401.469293:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829401.469298:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829401.469305:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327340672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829401.469311:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829401.469313:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829401.469318:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327340672/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829401.469329:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829401.469334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829401.469336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829401.469337:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829401.469340:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829401.469342:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829401.469347:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3549
00000400:00000200:1.0:1787829401.469350:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829401.469354:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829401.469357:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829401.469360:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469364:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.469368:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469371:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829401.469373:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829401.469374:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829401.469376:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829401.469378:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829401.469381:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829401.469383:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:0.1:1787829401.469387:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829401.469388:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469391:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829401.469394:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469397:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829401.469397:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000020:00000040:1.0:1787829401.469402:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829401.469408:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829401.469413:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829401.469414:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829401.469418:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829401.469420:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469421:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829401.469423:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829401.469426:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469430:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829401.469434:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829401.469437:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829401.469439:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829401.469442:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829401.469447:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829401.469449:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829401.469451:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829401.469455:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829401.469459:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829401.469467:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327340800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829401.469483:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829401.469485:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829401.469488:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327340800/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829401.469496:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829401.469500:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3559)
00000800:00000200:0.1:1787829401.469846:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829401.469848:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829401.470039:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829401.470041:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829401.470312:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829401.470314:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787829402.172793:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298222252, at 4298222272
00000100:00000040:3.0:1787829402.172793:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298222252, at 4298222272
00000100:00000010:2.0:1787829402.172813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000010:3.0:1787829402.172816:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000020:00000001:2.0:1787829402.300829:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829402.300832:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829402.300835:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829403.004783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829403.004794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829403.004797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829403.004806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829403.004812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829403.004817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829403.004820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829403.004824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829403.004829:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829404.028804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829404.028814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829404.028816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829404.028823:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829404.028827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829404.028831:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829404.028834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829404.028837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829404.028841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829405.052723:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829405.052736:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829405.052739:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829405.052744:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829405.052747:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829405.052751:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829405.052754:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829405.052756:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829405.052759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829405.372888:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829405.372892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829405.372896:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829406.076788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829406.076797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829406.076799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829406.076804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829406.076807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829406.076810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829406.076813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829406.076815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829406.076817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.589005:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829406.589013:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829406.589018:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829406.589034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589056:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829406.589059:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.589066:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829406.589070:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829406.589076:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829406.589086:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327340928:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829406.589091:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829406.589097:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829406.589101:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.589104:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829406.589110:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829406.589117:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829406.589128:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829406.589133:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829406.589146:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829406.589158:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327340928, portal 25
00000100:00000001:1.0:1787829406.589163:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829406.589166:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829406.589179:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589213:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829406.589223:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829406.589232:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829406.589242:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327340928, offset 0
00000400:00000200:1.0:1787829406.589260:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829406.589277:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.589288:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829406.589300:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829406.589309:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829406.589317:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829406.589324:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829406.589334:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40435:40435:256:4294967295] 192.168.203.116@tcp LPNI seq info [40435:40435:8:4294967295]
00000400:00000200:1.0:1787829406.589348:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.589384:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829406.589400:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.589411:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829406.589434:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829406.589444:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829406.589459:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829406.589470:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829406.589482:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.589488:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829406.589501:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829406.589516:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829406.589525:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829406.589532:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829406.589541:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829406.589550:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589564:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.589582:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589591:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.589593:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829406.589601:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829406.589606:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829406.589609:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298227372
00000100:00000010:1.0:1787829406.589615:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e634e0.
00000100:00000001:1.0:1787829406.589623:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829406.589629:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298227372
00000100:00000010:1.0:1787829406.589632:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63720.
00000100:00000001:1.0:1787829406.589636:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829406.589646:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829406.589650:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.589652:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829406.589679:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406]  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829406.589691:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829406.589693:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829406.589695:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829406.589698:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829406.589700:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829406.589704:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829406.589707:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829406.589716:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589723:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829406.589725:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.589728:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.589730:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829406.589734:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589742:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829406.589747:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589754:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829406.589757:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.589759:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.589760:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.589763:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829406.589765:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829406.589768:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829406.589773:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829406.589776:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829406.589781:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829406.589790:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327340928:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829406.589796:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.589799:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.589803:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327340928/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589810:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829406.589813:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829406.589816:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829406.589819:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829406.589822:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829406.589823:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829406.589826:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829406.589830:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829406.589834:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829406.589837:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829406.589839:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829406.589843:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829406.589844:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.589847:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.589856:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829406.589859:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.589862:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829406.589863:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829406.589865:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829406.589870:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589878:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829406.589880:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.589882:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829406.589884:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829406.589887:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829406.589890:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341056:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829406.589893:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829406.589895:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829406.589897:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.589899:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829406.589903:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829406.589905:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829406.589909:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829406.589911:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829406.589915:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829406.589919:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341056, portal 10
00000100:00000001:1.0:1787829406.589921:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829406.589923:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829406.589930:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.589938:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829406.589941:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829406.589944:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829406.589947:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327341056, offset 0
00000400:00000200:1.0:1787829406.589951:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829406.589955:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.589959:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829406.589964:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829406.589966:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829406.589969:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829406.589972:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829406.589975:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40436:40436:256:4294967295] 192.168.203.116@tcp LPNI seq info [40436:40436:8:4294967295]
00000400:00000200:1.0:1787829406.589980:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.589987:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829406.589993:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.589997:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829406.590002:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829406.590005:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829406.590010:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829406.590013:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829406.590018:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.590020:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829406.590023:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829406.590027:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829406.590030:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829406.590033:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829406.590035:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829406.590040:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590049:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.590053:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590060:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590062:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829406.590066:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829406.590070:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829406.590074:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829406.590077:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298226699
00000100:00000001:1.0:1787829406.590080:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298227372
00000100:00000001:1.0:1787829406.590082:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298226699
00000100:00000001:1.0:1787829406.590085:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298227372
00000100:00000001:1.0:1787829406.590087:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829406.590090:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590092:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829406.590095:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406]  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829406.590103:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829406.590105:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829406.590107:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829406.590109:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829406.590111:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829406.590113:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829406.590117:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829406.590126:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590133:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829406.590135:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590136:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.590137:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829406.590141:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590148:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829406.590151:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590157:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829406.590159:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590160:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.590161:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590162:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829406.590164:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829406.590165:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829406.590167:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829406.590168:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829406.590171:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829406.590176:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341056:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829406.590178:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829406.590181:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590185:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829406.590186:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.590188:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829406.590189:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829406.590190:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829406.590192:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341184:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829406.590194:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829406.590196:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829406.590197:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.590198:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829406.590201:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829406.590202:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829406.590205:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829406.590207:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829406.590210:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829406.590212:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341184, portal 4
00000100:00000001:1.0:1787829406.590214:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829406.590215:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829406.590219:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590224:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829406.590225:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829406.590227:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829406.590229:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327341184, offset 0
00000400:00000200:1.0:1787829406.590232:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829406.590236:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.590240:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829406.590242:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829406.590245:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829406.590248:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829406.590250:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829406.590254:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40437:40437:256:4294967295] 192.168.203.116@tcp LPNI seq info [40437:40437:8:4294967295]
00000400:00000200:1.0:1787829406.590258:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.590266:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829406.590271:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.590275:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829406.590279:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829406.590282:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829406.590287:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829406.590289:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829406.590294:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.590296:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829406.590300:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829406.590303:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829406.590306:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829406.590309:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829406.590312:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829406.590316:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590325:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.590328:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590336:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590338:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829406.590341:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829406.590345:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829406.590349:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829406.590351:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298226706
00000100:00000001:1.0:1787829406.590354:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298227372
00000100:00000001:1.0:1787829406.590357:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298226706
00000100:00000001:1.0:1787829406.590359:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298227372
00000100:00000001:1.0:1787829406.590361:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829406.590364:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590365:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829406.590369:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406]  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829406.590377:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829406.590379:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829406.590380:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829406.590383:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829406.590385:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829406.590387:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829406.590391:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829406.590400:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590412:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829406.590413:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590414:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.590415:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829406.590418:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590423:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829406.590426:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590430:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829406.590431:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590432:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.590433:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590435:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829406.590436:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829406.590438:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829406.590440:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829406.590441:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829406.590443:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829406.590448:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341184:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829406.590451:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829406.590455:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590461:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829406.590462:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.590465:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829406.590467:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829406.590469:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829406.590473:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341312:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829406.590476:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829406.590478:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829406.590480:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829406.590482:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829406.590485:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829406.590488:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829406.590491:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829406.590494:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829406.590498:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829406.590501:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341312, portal 4
00000100:00000001:1.0:1787829406.590504:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829406.590505:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829406.590512:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590520:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829406.590522:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829406.590525:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829406.590528:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327341312, offset 0
00000400:00000200:1.0:1787829406.590533:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829406.590538:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.590542:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829406.590545:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829406.590547:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829406.590550:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829406.590553:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829406.590556:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40438:40438:256:4294967295] 192.168.203.116@tcp LPNI seq info [40438:40438:8:4294967295]
00000400:00000200:1.0:1787829406.590561:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829406.590581:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829406.590587:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.590591:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829406.590595:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829406.590598:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829406.590603:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829406.590606:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829406.590610:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829406.590613:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829406.590616:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829406.590619:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829406.590623:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829406.590625:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829406.590628:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829406.590632:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590641:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.590645:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590652:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590653:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829406.590677:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829406.590681:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829406.590685:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829406.590687:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298226712
00000100:00000001:1.0:1787829406.590690:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298227372
00000100:00000001:1.0:1787829406.590693:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298226712
00000100:00000001:1.0:1787829406.590695:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298227372
00000100:00000001:1.0:1787829406.590697:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829406.590699:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590701:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829406.590705:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406]  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829406.590715:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829406.590716:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829406.590718:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829406.590721:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829406.590723:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829406.590725:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829406.590728:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829406.590738:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590745:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829406.590746:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590748:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.590750:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829406.590753:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590760:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829406.590764:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590771:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829406.590773:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590775:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829406.590776:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590779:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829406.590780:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829406.590783:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829406.590786:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829406.590787:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829406.590791:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829406.590796:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341312:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829406.590799:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590800:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.590802:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327341056/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590806:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829406.590807:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829406.590808:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829406.590810:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829406.590811:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829406.590812:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829406.590814:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829406.590815:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829406.590817:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829406.590818:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829406.590819:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829406.590821:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829406.590822:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590823:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.590825:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327341184/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590830:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829406.590831:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829406.590832:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829406.590833:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829406.590834:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829406.590834:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829406.590835:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829406.590836:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829406.590838:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829406.590839:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829406.590840:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829406.590841:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829406.590842:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590843:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829406.590845:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327341312/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829406.590849:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829406.590850:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829406.590851:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829406.590852:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829406.590853:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829406.590854:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829406.590855:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829406.590855:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:1.0:1787829406.590856:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829406.590857:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829406.590858:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829406.590859:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829406.590860:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590861:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829406.590864:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829406.590866:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829406.590867:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829406.590869:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829407.100778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829407.100785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829407.100787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829407.100792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829407.100795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829407.100799:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829407.100802:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829407.100804:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829407.100807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829407.292771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298227372, at 4298227392
00000100:00000040:3.0:1787829407.292771:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298227372, at 4298227392
00000100:00000010:3.0:1787829407.292789:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63720.
00000100:00000010:2.0:1787829407.292792:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e634e0.
00000100:00000001:2.0:1787829408.124748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829408.124754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829408.124756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829408.124760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829408.124763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829408.124766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829408.124768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829408.124770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829408.124772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829408.444863:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829408.444868:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829408.444871:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829409.148814:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829409.148825:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829409.148827:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829409.148835:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829409.148840:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829409.148845:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829409.148849:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829409.148852:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829409.148856:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829410.172793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829410.172801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829410.172804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829410.172812:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829410.172817:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829410.172825:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829410.172830:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829410.172834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829410.172839:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829411.196780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829411.196787:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829411.196789:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829411.196795:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829411.196799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829411.196802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829411.196805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829411.196809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829411.196812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829411.516844:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829411.516849:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829411.516853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829411.708923:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829411.708930:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829411.708933:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829411.708942:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.708955:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829411.708958:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.708961:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829411.708965:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829411.708968:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829411.708975:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327341440:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829411.708982:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829411.708987:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829411.708991:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.708996:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829411.709002:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829411.709006:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829411.709012:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829411.709018:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829411.709024:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829411.709033:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341440, portal 25
00000100:00000001:1.0:1787829411.709036:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829411.709039:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829411.709050:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709068:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829411.709077:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829411.709080:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829411.709085:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327341440, offset 0
00000400:00000200:1.0:1787829411.709095:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829411.709106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.709115:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829411.709122:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829411.709130:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829411.709137:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829411.709142:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829411.709149:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40439:40439:256:4294967295] 192.168.203.116@tcp LPNI seq info [40439:40439:8:4294967295]
00000400:00000200:1.0:1787829411.709153:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.709164:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829411.709171:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.709175:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829411.709184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829411.709190:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829411.709199:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829411.709216:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829411.709222:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.709226:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829411.709232:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829411.709237:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829411.709245:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829411.709249:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829411.709253:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829411.709259:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709280:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.709284:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709304:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709307:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829411.709314:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829411.709321:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829411.709326:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298232492
00000100:00000010:1.0:1787829411.709333:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63c60.
00000100:00000001:1.0:1787829411.709343:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829411.709350:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298232492
00000100:00000010:1.0:1787829411.709355:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e636c0.
00000100:00000001:1.0:1787829411.709361:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829411.709376:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829411.709380:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709385:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829411.709392:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411]  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829411.709406:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829411.709409:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829411.709414:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829411.709418:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829411.709421:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829411.709426:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829411.709432:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829411.709448:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709459:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829411.709461:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709466:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.709468:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829411.709475:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709488:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829411.709497:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709506:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829411.709509:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709510:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.709511:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709512:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829411.709514:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829411.709516:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829411.709519:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829411.709521:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829411.709523:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829411.709529:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327341440:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829411.709546:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709548:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.709550:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327341440/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709555:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829411.709557:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829411.709559:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829411.709561:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829411.709562:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829411.709563:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829411.709565:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829411.709567:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829411.709570:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829411.709571:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829411.709572:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829411.709574:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829411.709575:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709577:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709582:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829411.709583:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709585:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829411.709586:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829411.709587:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829411.709589:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709594:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829411.709595:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.709596:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829411.709598:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829411.709599:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829411.709601:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341568:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829411.709603:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829411.709604:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829411.709605:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.709607:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829411.709609:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829411.709611:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829411.709613:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829411.709615:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829411.709617:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829411.709619:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341568, portal 10
00000100:00000001:1.0:1787829411.709620:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829411.709621:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829411.709626:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709630:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829411.709632:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829411.709633:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829411.709635:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327341568, offset 0
00000400:00000200:1.0:1787829411.709637:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829411.709640:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.709642:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829411.709644:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829411.709646:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829411.709648:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829411.709649:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829411.709652:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40440:40440:256:4294967295] 192.168.203.116@tcp LPNI seq info [40440:40440:8:4294967295]
00000400:00000200:1.0:1787829411.709675:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.709680:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829411.709683:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.709686:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829411.709695:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829411.709697:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829411.709702:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829411.709704:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829411.709707:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.709708:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829411.709710:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829411.709713:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829411.709715:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829411.709716:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829411.709718:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829411.709720:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709726:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.709727:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709732:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709733:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829411.709735:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829411.709738:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829411.709740:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829411.709741:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298231819
00000100:00000001:1.0:1787829411.709743:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298232492
00000100:00000001:1.0:1787829411.709745:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298231819
00000100:00000001:1.0:1787829411.709746:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298232492
00000100:00000001:1.0:1787829411.709748:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829411.709749:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709750:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829411.709753:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411]  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829411.709757:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829411.709758:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829411.709760:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829411.709761:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829411.709762:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829411.709764:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829411.709766:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829411.709772:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709776:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829411.709777:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709778:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.709779:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829411.709781:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709785:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829411.709787:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709791:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829411.709793:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709794:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.709794:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709796:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829411.709797:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829411.709798:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829411.709801:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829411.709802:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829411.709805:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829411.709810:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341568:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829411.709812:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829411.709814:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709822:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829411.709823:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.709825:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829411.709826:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829411.709827:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829411.709829:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341696:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829411.709831:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829411.709832:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829411.709833:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.709834:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829411.709836:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829411.709837:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829411.709839:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829411.709841:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829411.709843:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829411.709845:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341696, portal 4
00000100:00000001:1.0:1787829411.709846:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829411.709847:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829411.709851:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709855:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829411.709856:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829411.709858:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829411.709859:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327341696, offset 0
00000400:00000200:1.0:1787829411.709862:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829411.709864:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.709866:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829411.709867:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829411.709869:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829411.709871:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829411.709872:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829411.709875:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40441:40441:256:4294967295] 192.168.203.116@tcp LPNI seq info [40441:40441:8:4294967295]
00000400:00000200:1.0:1787829411.709878:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.709882:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829411.709885:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.709887:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829411.709890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829411.709891:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829411.709894:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829411.709896:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829411.709898:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.709899:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829411.709901:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829411.709903:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829411.709905:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829411.709906:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829411.709907:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829411.709910:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709915:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.709917:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709921:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709922:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829411.709923:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829411.709925:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829411.709927:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829411.709928:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298231826
00000100:00000001:1.0:1787829411.709929:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298232492
00000100:00000001:1.0:1787829411.709931:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298231826
00000100:00000001:1.0:1787829411.709932:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298232492
00000100:00000001:1.0:1787829411.709934:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829411.709936:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.709937:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829411.709940:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411]  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829411.709945:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829411.709946:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829411.709946:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829411.709948:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829411.709949:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829411.709950:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829411.709952:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829411.709957:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709961:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829411.709962:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709963:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.709964:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829411.709966:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709970:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829411.709972:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.709976:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829411.709977:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709978:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.709979:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.709980:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829411.709981:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829411.709982:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829411.709985:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829411.709986:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829411.709989:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829411.709994:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341696:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829411.709996:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829411.709998:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710001:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829411.710002:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.710003:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829411.710005:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829411.710006:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829411.710007:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341824:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829411.710009:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829411.710010:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829411.710011:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829411.710012:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829411.710014:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829411.710015:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829411.710017:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829411.710018:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829411.710020:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829411.710021:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341824, portal 4
00000100:00000001:1.0:1787829411.710023:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829411.710023:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829411.710027:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710031:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829411.710033:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829411.710035:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829411.710036:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327341824, offset 0
00000400:00000200:1.0:1787829411.710038:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829411.710041:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.710043:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829411.710044:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829411.710046:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829411.710047:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829411.710049:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829411.710051:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40442:40442:256:4294967295] 192.168.203.116@tcp LPNI seq info [40442:40442:8:4294967295]
00000400:00000200:1.0:1787829411.710054:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829411.710058:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829411.710060:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.710062:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829411.710064:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829411.710066:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829411.710068:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829411.710070:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829411.710072:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829411.710074:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829411.710075:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829411.710077:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829411.710079:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829411.710080:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829411.710081:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829411.710083:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710088:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.710090:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710094:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.710096:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829411.710097:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829411.710099:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829411.710101:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829411.710102:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298231831
00000100:00000001:1.0:1787829411.710104:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298232492
00000100:00000001:1.0:1787829411.710105:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298231831
00000100:00000001:1.0:1787829411.710107:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298232492
00000100:00000001:1.0:1787829411.710108:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829411.710109:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.710110:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829411.710112:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411]  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829411.710116:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829411.710117:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829411.710118:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829411.710119:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829411.710120:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829411.710122:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829411.710124:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829411.710129:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710133:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829411.710134:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710135:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.710135:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829411.710137:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710141:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829411.710144:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710148:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829411.710150:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710151:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829411.710152:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710153:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829411.710154:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829411.710155:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829411.710157:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829411.710158:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829411.710160:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829411.710165:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327341824:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829411.710167:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710168:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.710170:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327341568/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710174:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829411.710175:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829411.710176:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829411.710177:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829411.710178:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829411.710179:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829411.710180:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829411.710181:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829411.710183:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829411.710184:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829411.710185:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829411.710186:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829411.710187:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710189:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.710191:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327341696/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710195:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829411.710196:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829411.710197:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829411.710199:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829411.710199:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829411.710200:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829411.710201:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829411.710202:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:1.0:1787829411.710203:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829411.710204:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829411.710204:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829411.710206:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829411.710207:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710208:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829411.710209:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327341824/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829411.710213:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829411.710214:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829411.710215:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829411.710216:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829411.710216:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829411.710217:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829411.710218:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829411.710219:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829411.710220:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829411.710221:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829411.710222:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829411.710223:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829411.710224:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710225:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829411.710226:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829411.710227:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829411.710229:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829411.710230:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829412.220741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829412.220746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829412.220747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829412.220751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829412.220753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829412.220755:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829412.220757:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829412.220759:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829412.220761:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829412.412742:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298232492, at 4298232512
00000100:00000040:3.0:1787829412.412747:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298232492, at 4298232512
00000100:00000010:2.0:1787829412.412756:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63c60.
00000100:00000010:3.0:1787829412.412759:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e636c0.
00000100:00000001:2.0:1787829413.244742:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829413.244748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829413.244749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829413.244753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829413.244756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829413.244758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829413.244760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829413.244762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829413.244764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829414.268753:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829414.268761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829414.268763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829414.268769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829414.268772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829414.268775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829414.268777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829414.268779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829414.268781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829414.588884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829414.588888:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829414.588892:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829415.292754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829415.292760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829415.292762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829415.292766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829415.292768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829415.292778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829415.292780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829415.292782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829415.292785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829416.316739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829416.316745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829416.316748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829416.316753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829416.316757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829416.316760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829416.316771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829416.316774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829416.316778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.828952:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829416.828958:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829416.828961:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829416.828969:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.828982:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829416.828985:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.828990:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829416.828995:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829416.828999:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829416.829006:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327341952:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829416.829013:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829416.829020:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829416.829024:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.829031:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829416.829039:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829416.829043:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829416.829053:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829416.829058:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829416.829065:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829416.829072:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327341952, portal 25
00000100:00000001:1.0:1787829416.829076:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829416.829078:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829416.829087:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829098:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829416.829104:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829416.829106:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829416.829110:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327341952, offset 0
00000400:00000200:1.0:1787829416.829118:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829416.829127:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.829133:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829416.829138:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829416.829144:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829416.829151:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829416.829156:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829416.829159:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40443:40443:256:4294967295] 192.168.203.116@tcp LPNI seq info [40443:40443:8:4294967295]
00000400:00000200:1.0:1787829416.829162:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.829172:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829416.829183:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.829189:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829416.829199:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829416.829203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829416.829213:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829416.829219:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829416.829229:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.829233:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829416.829239:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829416.829246:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829416.829254:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829416.829258:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829416.829263:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829416.829268:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829283:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.829287:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829300:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.829303:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829416.829309:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829416.829315:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829416.829318:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298237612
00000100:00000010:1.0:1787829416.829325:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63660.
00000100:00000001:1.0:1787829416.829333:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829416.829339:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298237612
00000100:00000010:1.0:1787829416.829344:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63a80.
00000100:00000001:1.0:1787829416.829347:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829416.829352:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829416.829356:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.829360:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829416.829367:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416]  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829416.829385:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829416.829388:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829416.829391:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829416.829395:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829416.829399:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829416.829402:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829416.829410:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829416.829429:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829440:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829416.829443:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829446:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.829450:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829416.829456:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829472:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829416.829480:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829490:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829416.829494:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829509:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.829512:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829516:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829416.829519:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829416.829524:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829416.829530:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829416.829535:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829416.829543:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829416.829564:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327341952:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829416.829576:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829582:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.829590:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327341952/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829604:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829416.829607:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829416.829609:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829416.829612:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829416.829613:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829416.829614:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829416.829616:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829416.829618:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829416.829621:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829416.829622:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829416.829623:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829416.829626:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829416.829627:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829628:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829634:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829416.829635:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.829637:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829416.829638:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829416.829639:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829416.829642:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829651:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829416.829653:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.829678:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829416.829680:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829416.829682:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829416.829684:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342080:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829416.829686:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829416.829688:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829416.829689:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.829696:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829416.829698:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829416.829699:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829416.829702:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829416.829704:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829416.829707:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829416.829710:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342080, portal 10
00000100:00000001:1.0:1787829416.829711:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829416.829712:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829416.829717:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829722:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829416.829724:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829416.829725:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829416.829727:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327342080, offset 0
00000400:00000200:1.0:1787829416.829729:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829416.829732:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.829735:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829416.829737:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829416.829739:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829416.829741:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829416.829743:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829416.829745:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40444:40444:256:4294967295] 192.168.203.116@tcp LPNI seq info [40444:40444:8:4294967295]
00000400:00000200:1.0:1787829416.829748:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.829753:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829416.829756:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.829758:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829416.829761:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829416.829763:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829416.829767:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829416.829769:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829416.829773:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.829775:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829416.829778:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829416.829782:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829416.829785:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829416.829787:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829416.829790:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829416.829794:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829803:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.829806:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829814:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.829816:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829416.829819:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829416.829823:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829416.829826:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829416.829828:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298236939
00000100:00000001:1.0:1787829416.829830:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298237612
00000100:00000001:1.0:1787829416.829831:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298236939
00000100:00000001:1.0:1787829416.829833:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298237612
00000100:00000001:1.0:1787829416.829834:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829416.829836:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.829838:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829416.829841:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416]  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829416.829847:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829416.829848:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829416.829849:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829416.829851:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829416.829852:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829416.829853:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829416.829855:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829416.829861:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829865:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829416.829866:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829867:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.829868:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829416.829870:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829874:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829416.829877:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829881:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829416.829883:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829884:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.829885:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.829886:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829416.829888:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829416.829889:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829416.829892:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829416.829893:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829416.829896:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829416.829901:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342080:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829416.829903:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829416.829905:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829909:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829416.829910:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.829911:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829416.829912:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829416.829914:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829416.829916:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342208:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829416.829917:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829416.829919:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829416.829920:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.829921:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829416.829923:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829416.829925:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829416.829927:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829416.829929:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829416.829931:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829416.829933:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342208, portal 4
00000100:00000001:1.0:1787829416.829934:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829416.829935:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829416.829939:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.829943:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829416.829945:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829416.829946:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829416.829948:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327342208, offset 0
00000400:00000200:1.0:1787829416.829950:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829416.829953:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.829955:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829416.829958:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829416.829959:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829416.829961:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829416.829963:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829416.829965:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40445:40445:256:4294967295] 192.168.203.116@tcp LPNI seq info [40445:40445:8:4294967295]
00000400:00000200:1.0:1787829416.829967:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.829972:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829416.829975:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.829977:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829416.829980:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829416.829982:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829416.829984:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829416.829986:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829416.829989:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.829990:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829416.829992:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829416.829995:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829416.829998:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829416.830000:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829416.830003:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829416.830007:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830014:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.830018:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830024:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.830026:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829416.830030:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829416.830033:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829416.830037:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829416.830039:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298236946
00000100:00000001:1.0:1787829416.830041:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298237612
00000100:00000001:1.0:1787829416.830043:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298236946
00000100:00000001:1.0:1787829416.830046:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298237612
00000100:00000001:1.0:1787829416.830048:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829416.830049:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.830051:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829416.830054:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416]  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829416.830059:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829416.830060:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829416.830062:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829416.830064:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829416.830065:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829416.830066:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829416.830068:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829416.830074:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830077:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829416.830078:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830079:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.830080:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829416.830082:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830086:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829416.830089:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830094:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829416.830095:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830097:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.830097:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830099:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829416.830100:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829416.830101:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829416.830103:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829416.830104:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829416.830106:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829416.830111:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342208:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829416.830113:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829416.830115:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830119:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829416.830120:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.830122:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829416.830123:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829416.830124:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829416.830126:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829416.830128:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829416.830129:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829416.830130:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829416.830131:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829416.830133:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829416.830134:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829416.830136:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829416.830138:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829416.830140:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829416.830142:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342336, portal 4
00000100:00000001:1.0:1787829416.830144:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829416.830144:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829416.830148:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830153:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829416.830154:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829416.830156:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829416.830157:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327342336, offset 0
00000400:00000200:1.0:1787829416.830160:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829416.830162:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.830164:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829416.830166:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829416.830168:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829416.830170:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829416.830171:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829416.830173:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40446:40446:256:4294967295] 192.168.203.116@tcp LPNI seq info [40446:40446:8:4294967295]
00000400:00000200:1.0:1787829416.830176:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829416.830180:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829416.830182:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.830184:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829416.830187:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829416.830189:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829416.830191:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829416.830193:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829416.830196:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829416.830197:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829416.830199:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829416.830201:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829416.830203:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:1.0:1787829416.830204:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:1.0:1787829416.830206:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829416.830208:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830213:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.830215:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830220:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.830221:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829416.830223:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829416.830224:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829416.830226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829416.830227:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298236951
00000100:00000001:1.0:1787829416.830229:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298237612
00000100:00000001:1.0:1787829416.830230:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298236951
00000100:00000001:1.0:1787829416.830232:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298237612
00000100:00000001:1.0:1787829416.830233:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829416.830234:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.830235:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829416.830237:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416]  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829416.830242:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829416.830242:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829416.830243:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829416.830245:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829416.830246:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829416.830247:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829416.830248:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829416.830253:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830262:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829416.830263:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830264:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.830265:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829416.830268:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830271:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829416.830274:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830278:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829416.830279:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830280:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829416.830281:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830282:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829416.830283:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829416.830284:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829416.830286:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829416.830287:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829416.830290:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829416.830294:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829416.830296:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830298:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.830300:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327342080/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830304:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829416.830305:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829416.830307:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829416.830309:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829416.830310:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829416.830311:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829416.830312:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829416.830313:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829416.830315:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829416.830316:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829416.830317:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829416.830319:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829416.830320:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830321:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.830323:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327342208/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830327:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829416.830328:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829416.830329:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829416.830330:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829416.830331:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829416.830331:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829416.830332:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829416.830333:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829416.830334:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829416.830335:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829416.830336:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829416.830337:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829416.830338:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830339:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829416.830341:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327342336/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829416.830344:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829416.830345:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829416.830346:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829416.830348:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829416.830348:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829416.830349:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829416.830350:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829416.830351:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829416.830353:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829416.830353:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829416.830354:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829416.830356:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829416.830356:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830357:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829416.830360:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829416.830360:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829416.830362:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829416.830363:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829417.340754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829417.340760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829417.340762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829417.340773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829417.340776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829417.340779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829417.340781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829417.340783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829417.340785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829417.532753:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298237612, at 4298237632
00000100:00000040:2.0:1787829417.532754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298237612, at 4298237632
00000100:00000010:3.0:1787829417.532768:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a80.
00000100:00000010:2.0:1787829417.532775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63660.
00000020:00000001:2.0:1787829417.660814:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829417.660817:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829417.660819:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829418.364713:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829418.364719:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829418.364720:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829418.364725:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829418.364727:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829418.364730:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829418.364732:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829418.364734:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829418.364736:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829419.388804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829419.388812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829419.388814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829419.388831:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829419.388835:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829419.388838:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829419.388841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829419.388844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829419.388846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829420.412741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829420.412751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829420.412753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829420.412761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829420.412766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829420.412771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829420.412774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829420.412778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829420.412782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829420.732835:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829420.732840:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829420.732844:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829421.436792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829421.436799:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829421.436802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829421.436807:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829421.436810:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829421.436814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829421.436816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829421.436819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829421.436821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.949080:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829421.949087:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829421.949092:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829421.949103:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.949131:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829421.949136:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.949143:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829421.949150:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829421.949157:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829421.949169:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327342464:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829421.949180:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829421.949187:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829421.949193:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.949199:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829421.949208:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829421.949215:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829421.949230:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829421.949238:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829421.949251:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829421.949264:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342464, portal 25
00000100:00000001:1.0:1787829421.949271:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829421.949277:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829421.949292:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.949315:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829421.949324:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829421.949331:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98770.
00000100:00000200:1.0:1787829421.949341:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327342464, offset 0
00000400:00000200:1.0:1787829421.949357:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829421.949381:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.949392:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829421.949403:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829421.949413:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829421.949418:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829421.949421:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829421.949425:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40447:40447:256:4294967295] 192.168.203.116@tcp LPNI seq info [40447:40447:8:4294967295]
00000400:00000200:1.0:1787829421.949431:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.949470:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829421.949488:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829421.949500:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829421.949515:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829421.949527:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829421.949553:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829421.949562:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829421.949567:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298242732
00000100:00000010:1.0:1787829421.949577:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63600.
00000100:00000001:1.0:1787829421.949588:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829421.949596:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298242732
00000100:00000010:1.0:1787829421.949603:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e633c0.
00000100:00000001:1.0:1787829421.949608:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829421.949620:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829421.949624:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.949628:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.949632:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829421.949642:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829421.949645:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829421.949647:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829421.949649:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829421.949651:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829421.949696:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.949706:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829421.949708:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.949711:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829421.949713:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829421.949716:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829421.949720:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342592:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829421.949724:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829421.949726:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829421.949728:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.949731:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829421.949735:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829421.949737:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829421.949742:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829421.949745:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd946304d0.
00000400:00000010:1.0:1787829421.949750:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829421.949753:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342592, portal 10
00000100:00000001:1.0:1787829421.949756:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829421.949757:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829421.949767:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.949775:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829421.949778:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829421.949781:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98a18.
00000100:00000200:1.0:1787829421.949783:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327342592, offset 0
00000400:00000200:1.0:1787829421.949788:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829421.949792:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.949797:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829421.949800:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829421.949803:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829421.949806:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829421.949809:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829421.949812:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40448:40448:255:4294967295] 192.168.203.116@tcp LPNI seq info [40448:40448:7:4294967295]
00000400:00000200:1.0:1787829421.949816:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.949823:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829421.949828:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829421.949831:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955200.
00000800:00000200:1.0:1787829421.949836:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829421.949840:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829421.949843:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829421.949847:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829421.949851:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829421.949853:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298242059
00000100:00000001:1.0:1787829421.949856:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298242732
00000100:00000001:1.0:1787829421.949858:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298242059
00000100:00000001:1.0:1787829421.949861:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298242732
00000100:00000001:1.0:1787829421.949863:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829421.949865:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.949868:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829421.949873:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.949881:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829421.949882:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.949885:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829421.949887:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829421.949889:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829421.949892:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342720:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829421.949899:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829421.949901:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829421.949903:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.949905:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829421.949909:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829421.949911:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829421.949915:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829421.949918:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829421.949922:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829421.949925:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342720, portal 4
00000100:00000001:1.0:1787829421.949927:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829421.949929:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829421.949936:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.949943:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829421.949946:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829421.949949:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98c38.
00000100:00000200:1.0:1787829421.949951:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327342720, offset 0
00000400:00000200:1.0:1787829421.949956:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829421.949960:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.949963:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829421.949966:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829421.949969:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829421.949972:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829421.949974:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829421.949977:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40449:40449:254:4294967295] 192.168.203.116@tcp LPNI seq info [40449:40449:6:4294967295]
00000400:00000200:1.0:1787829421.949982:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.949988:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829421.949993:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829421.949997:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829421.950002:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829421.950007:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829421.950010:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829421.950013:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829421.950016:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829421.950019:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298242066
00000100:00000001:1.0:1787829421.950022:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298242732
00000100:00000001:1.0:1787829421.950024:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298242066
00000100:00000001:1.0:1787829421.950027:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298242732
00000100:00000001:1.0:1787829421.950029:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829421.950031:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.950033:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829421.950037:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.950044:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829421.950046:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.950048:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829421.950050:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829421.950052:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829421.950055:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829421.950059:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829421.950061:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829421.950062:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829421.950064:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829421.950068:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7000.
02000000:00000001:1.0:1787829421.950070:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829421.950073:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829421.950076:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630d68.
00000400:00000010:1.0:1787829421.950080:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829421.950083:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327342848, portal 4
00000100:00000001:1.0:1787829421.950085:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829421.950087:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829421.950093:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829421.950101:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829421.950104:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829421.950106:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98b28.
00000100:00000200:1.0:1787829421.950109:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327342848, offset 0
00000400:00000200:1.0:1787829421.950113:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829421.950117:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.950121:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829421.950124:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829421.950126:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829421.950129:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829421.950132:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829421.950135:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40450:40450:253:4294967295] 192.168.203.116@tcp LPNI seq info [40450:40450:5:4294967295]
00000400:00000200:1.0:1787829421.950139:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829421.950145:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829421.950150:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829421.950155:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955900.
00000800:00000200:1.0:1787829421.950159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829421.950162:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829421.950165:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829421.950168:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829421.950171:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829421.950173:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298242071
00000100:00000001:1.0:1787829421.950176:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298242732
00000100:00000001:1.0:1787829421.950178:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298242071
00000100:00000001:1.0:1787829421.950180:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298242732
00000100:00000001:1.0:1787829421.950182:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829421.950185:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.950188:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.950190:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829421.950193:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829421.950196:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829421.950198:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829421.950199:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829421.950201:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829421.950203:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829422.460723:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829422.460731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829422.460733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829422.460739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829422.460742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829422.460746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829422.460748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829422.460751:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829422.460753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829422.652740:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298242732, at 4298242752
00000100:00000040:3.0:1787829422.652740:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298242732, at 4298242752
00000100:00000010:2.0:1787829422.652758:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63600.
00000100:00000010:3.0:1787829422.652761:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e633c0.
00000100:00000001:2.0:1787829423.484823:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829423.484832:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829423.484835:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829423.484841:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829423.484846:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829423.484851:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829423.484854:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829423.484858:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829423.484862:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829423.804922:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829423.804926:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829423.804929:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829425.021059:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829425.021118:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829425.021131:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829425.021136:0:2336:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829425.021142:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829425.021149:0:2336:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829425.021155:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829425.021159:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:2.0:1787829425.021167:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829425.021172:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829425.021181:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98770
00000400:00000010:2.0:1787829425.021184:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98770.
00000100:00000001:2.0:1787829425.021191:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829425.021203:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021236:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829425.021241:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021250:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829425.021253:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829425.021265:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829425.021267:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955200.
00000400:00000200:2.0:1787829425.021271:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829425.021274:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829425.021277:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98a18
00000400:00000010:2.0:1787829425.021279:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98a18.
00000100:00000001:2.0:1787829425.021283:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829425.021288:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021297:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829425.021300:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021308:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829425.021310:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829425.021315:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829425.021317:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:2.0:1787829425.021319:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829425.021321:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829425.021324:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98c38
00000400:00000010:2.0:1787829425.021325:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98c38.
00000100:00000001:2.0:1787829425.021328:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829425.021331:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021337:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829425.021339:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021345:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829425.021347:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829425.021352:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829425.021360:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955900.
00000400:00000200:2.0:1787829425.021364:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829425.021367:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829425.021370:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98b28
00000400:00000010:2.0:1787829425.021373:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98b28.
00000100:00000001:2.0:1787829425.021375:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829425.021381:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021389:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829425.021392:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829425.021399:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829425.021401:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829425.021640:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829425.021646:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829425.021651:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829425.021705:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829425]  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829425.021723:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829425.021726:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829425.021731:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd946304d0.
00000400:00000200:1.0:1787829425.021737:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:1.0:1787829425.021739:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:1.0:1787829425.021744:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829425.021750:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829425.021761:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.021769:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829425.021771:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.021775:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.021778:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829425.021782:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.021790:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829425.021796:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.021804:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829425.021807:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.021810:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.021812:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.021815:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829425.021819:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829425.021823:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829425.021829:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829425.021833:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829425.021838:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829425.021848:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342592:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829425.021853:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829425.021857:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829425]  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829425.021866:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829425.021868:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829425.021870:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829425.021874:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829425.021876:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829425.021881:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829425.021886:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829425.021895:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.021902:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829425.021904:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.021905:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.021907:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829425.021911:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.021925:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829425.021929:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.021936:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829425.021939:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.021941:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.021942:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.021944:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829425.021946:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829425.021948:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829425.021953:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829425.021956:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829425.021967:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829425.021976:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342720:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829425.021983:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829425.021987:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829425]  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829425.021995:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829425.021998:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829425.022000:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630d68.
00000400:00000200:1.0:1787829425.022003:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829425.022005:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829425.022007:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829425.022010:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829425.022024:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022031:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829425.022032:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022034:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.022035:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829425.022039:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022046:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829425.022049:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022055:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829425.022058:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022059:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.022060:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022062:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829425.022064:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829425.022072:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829425.022075:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829425.022078:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829425.022082:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829425.022089:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327342848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829425.022094:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829425.022098:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829425]  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829425.022112:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829425.022113:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829425.022116:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829425.022119:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829425.022121:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829425.022124:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829425.022127:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829425.022135:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022142:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829425.022143:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022146:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.022147:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829425.022151:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022158:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829425.022162:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022169:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829425.022171:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022173:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829425.022174:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022176:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829425.022196:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829425.022200:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829425.022204:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829425.022207:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829425.022211:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829425.022221:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327342464:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829425.022225:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022233:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829425.022238:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327342592/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022246:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829425.022250:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829425.022256:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:1.0:1787829425.022261:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829425.022264:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829425.022267:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829425.022270:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829425.022273:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829425.022279:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829425.022281:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829425.022283:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829425.022287:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829425.022289:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022291:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829425.022295:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327342720/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022303:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829425.022304:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829425.022306:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829425.022309:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829425.022310:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829425.022312:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829425.022314:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829425.022315:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829425.022317:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829425.022319:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829425.022320:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829425.022324:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829425.022326:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022328:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829425.022332:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327342848/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022339:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829425.022340:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829425.022342:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7000.
02000000:00000001:1.0:1787829425.022345:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829425.022346:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829425.022348:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829425.022350:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829425.022352:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:1.0:1787829425.022355:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829425.022356:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829425.022357:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829425.022360:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829425.022362:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022364:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829425.022367:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327342464/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829425.022375:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829425.022376:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829425.022378:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5800.
02000000:00000001:1.0:1787829425.022380:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829425.022381:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829425.022383:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829425.022384:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829425.022385:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829425.022387:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829425.022388:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829425.022389:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829425.022391:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829425.022391:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022394:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829425.022402:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829425.022404:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829425.022406:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829425.022408:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829426.876866:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829426.876870:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829426.876873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829427.068786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829427.068801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829427.068805:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829427.068809:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829427.068816:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829427.068827:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3574
00000400:00000200:1.0:1787829427.068839:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829427.068848:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829427.068854:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829427.068859:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.068863:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.068867:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.068870:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829427.068875:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829427.068878:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829427.068882:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829427.068885:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829427.068887:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829427.068889:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829427.068892:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.068894:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.068895:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.068896:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829427.068899:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829427.068902:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829427.068903:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829427.068906:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829427.068909:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829427.068912:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.068914:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829427.068917:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829427.068923:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829427.068926:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829427.068930:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829427.068932:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.068935:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829427.068937:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829427.068942:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829427.068951:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829427.068966:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829427.068977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327342976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829427.068988:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829427.068994:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829427.069000:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327342976/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829427.069029:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829427.069033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829427.069037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829427.069039:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829427.069041:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829427.069044:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829427.069047:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3574
00000400:00000200:1.0:1787829427.069052:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829427.069055:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829427.069057:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829427.069060:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069062:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.069064:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069067:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829427.069069:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829427.069071:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829427.069072:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829427.069075:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829427.069077:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829427.069079:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829427.069083:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069085:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.069087:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069089:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829427.069092:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829427.069094:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069097:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829427.069098:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829427.069105:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829427.069107:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069109:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829427.069114:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829427.069118:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069120:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829427.069124:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829427.069127:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829427.069129:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829427.069131:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069133:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829427.069135:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829427.069137:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829427.069141:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829427.069154:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829427.069162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327343104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829427.069185:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829427.069190:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829427.069199:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327343104/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829427.069214:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829427.069217:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829427.069220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829427.069222:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829427.069223:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829427.069226:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829427.069229:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3574
00000400:00000200:1.0:1787829427.069233:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829427.069236:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829427.069238:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829427.069241:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069243:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.069245:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069247:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829427.069249:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829427.069251:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829427.069252:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829427.069254:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829427.069257:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829427.069259:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829427.069264:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069266:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.069269:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069274:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829427.069282:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829427.069291:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069297:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829427.069301:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829427.069307:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829427.069310:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069315:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829427.069320:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829427.069326:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069329:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829427.069335:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829427.069340:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829427.069345:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829427.069348:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069352:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829427.069354:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829427.069358:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829427.069366:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829427.069373:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829427.069379:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327343232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829427.069394:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829427.069399:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829427.069408:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327343232/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829427.069447:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829427.069452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829427.069459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829427.069463:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829427.069465:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829427.069469:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829427.069474:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3574
00000400:00000200:1.0:1787829427.069481:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829427.069485:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829427.069488:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829427.069495:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069499:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.069501:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069506:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829427.069511:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829427.069518:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829427.069522:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829427.069525:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829427.069529:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829427.069533:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829427.069540:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069545:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829427.069551:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069557:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829427.069563:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829427.069571:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829427.069577:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829427.069581:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829427.069585:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829427.069589:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069594:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829427.069598:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829427.069603:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069606:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829427.069612:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829427.069617:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829427.069621:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829427.069625:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829427.069627:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829427.069631:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829427.069635:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829427.069642:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829427.069648:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829427.069650:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829427.069684:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00100000:1.0:1787829427.069718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327343360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829427.069732:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829427.069738:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829427.069746:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327343360/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829427.069765:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829427.069769:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3584)
00000800:00000200:0.1:1787829427.070241:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829427.070244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829427.070584:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829427.070587:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829427.070934:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829427.070936:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1787829427.772724:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298247853, at 4298247872
00000100:00000040:3.0:1787829427.772724:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298247853, at 4298247872
00000100:00000010:2.0:1787829427.772739:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000100:00000010:3.0:1787829427.772742:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000020:00000001:2.0:1787829429.948878:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829429.948883:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829429.948888:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829432.188793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829432.188803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829432.188806:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829432.188809:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829432.188814:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829432.188820:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3579
00000400:00000200:1.0:1787829432.188829:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829432.188836:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829432.188839:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829432.188843:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188847:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.188850:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188852:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829432.188856:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829432.188858:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829432.188861:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829432.188865:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829432.188867:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829432.188870:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829432.188873:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188874:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.188876:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188877:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829432.188879:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829432.188881:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188883:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829432.188885:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829432.188887:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829432.188889:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.188890:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829432.188893:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829432.188900:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829432.188902:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829432.188905:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829432.188906:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.188908:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829432.188910:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829432.188913:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829432.188919:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829432.188924:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829432.188932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327343488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829432.188940:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829432.188943:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829432.188946:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327343488/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829432.188964:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829432.188967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829432.188969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829432.188971:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829432.188972:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829432.188974:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829432.188976:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3579
00000400:00000200:1.0:1787829432.188979:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829432.188981:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829432.188982:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829432.188984:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188986:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.188987:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.188988:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829432.188990:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829432.188992:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829432.188993:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829432.188995:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829432.188997:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829432.188998:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829432.189000:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189002:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.189003:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189005:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829432.189008:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829432.189010:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189012:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829432.189014:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829432.189018:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829432.189020:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189021:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829432.189024:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829432.189027:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189032:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829432.189035:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829432.189039:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829432.189044:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829432.189046:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189047:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829432.189050:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829432.189054:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829432.189058:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829432.189067:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829432.189080:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327343616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829432.189088:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829432.189096:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829432.189111:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327343616/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829432.189132:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829432.189140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829432.189147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829432.189152:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829432.189155:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829432.189160:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829432.189164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3579
00000400:00000200:1.0:1787829432.189172:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829432.189174:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829432.189177:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829432.189183:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189186:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.189192:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189195:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829432.189196:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829432.189197:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829432.189199:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829432.189202:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829432.189208:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829432.189214:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829432.189218:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189220:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.189222:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189223:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829432.189228:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829432.189236:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189239:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829432.189243:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829432.189246:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829432.189249:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189257:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829432.189261:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829432.189264:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189270:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829432.189279:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829432.189291:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829432.189296:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829432.189299:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189309:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829432.189313:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829432.189321:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829432.189331:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829432.189341:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829432.189357:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327343744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829432.189378:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829432.189380:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829432.189400:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327343744/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829432.189422:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829432.189430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829432.189443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829432.189445:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829432.189448:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829432.189454:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829432.189459:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3579
00000400:00000200:1.0:1787829432.189475:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829432.189482:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829432.189492:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829432.189504:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189512:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829432.189520:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189524:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829432.189530:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829432.189532:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829432.189534:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829432.189539:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829432.189549:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829432.189557:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829432.189562:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829432.189565:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829432.189568:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787829432.189575:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1787829432.189579:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189583:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829432.189591:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829432.189608:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829432.189611:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829432.189615:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829432.189619:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829432.189623:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189627:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829432.189629:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829432.189637:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189641:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829432.189695:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829432.189701:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829432.189709:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829432.189711:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829432.189720:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829432.189724:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829432.189729:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829432.189742:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829432.189753:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829432.189764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327343872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829432.189783:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829432.189788:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829432.189795:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327343872/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829432.189807:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829432.189813:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3589)
00000800:00000200:0.1:1787829432.190093:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829432.190095:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829432.190326:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829432.190328:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829432.190541:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829432.190543:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1787829432.892793:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298252973, at 4298252992
00000100:00000040:3.0:1787829432.892793:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298252973, at 4298252992
00000100:00000010:2.0:1787829432.892813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000100:00000010:3.0:1787829432.892815:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000020:00000001:2.0:1787829433.020856:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829433.020861:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829433.020866:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829436.092833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829436.092837:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829436.092840:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829437.308814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829437.308826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829437.308830:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829437.308835:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829437.308840:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829437.308848:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3584
00000400:00000200:3.0:1787829437.308857:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829437.308865:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829437.308869:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829437.308872:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.308874:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.308885:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.308887:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829437.308890:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829437.308892:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829437.308894:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829437.308897:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829437.308899:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829437.308901:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829437.308903:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.308904:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.308906:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.308907:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829437.308909:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829437.308912:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829437.308913:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829437.308916:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829437.308919:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829437.308922:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.308924:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829437.308926:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829437.308935:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829437.308938:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829437.308942:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829437.308944:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.308947:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829437.308949:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829437.308953:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829437.308962:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829437.308968:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829437.308978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327344000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829437.308989:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829437.308994:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829437.308998:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327344000/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829437.309019:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829437.309022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829437.309024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829437.309025:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829437.309027:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829437.309029:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829437.309031:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3584
00000400:00000200:3.0:1787829437.309034:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829437.309036:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829437.309038:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829437.309040:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309041:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.309043:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309045:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829437.309047:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829437.309048:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829437.309049:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829437.309052:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829437.309053:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829437.309054:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829437.309057:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309059:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.309060:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309062:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829437.309064:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829437.309069:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309073:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829437.309077:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829437.309082:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829437.309086:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309088:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829437.309093:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829437.309101:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309120:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829437.309128:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829437.309133:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829437.309137:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829437.309141:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309147:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829437.309152:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829437.309157:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829437.309167:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829437.309175:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829437.309186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327344128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829437.309216:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829437.309221:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829437.309228:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327344128/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829437.309241:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829437.309244:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829437.309250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829437.309253:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829437.309255:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829437.309259:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829437.309265:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3584
00000400:00000200:3.0:1787829437.309271:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829437.309277:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829437.309282:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829437.309289:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309294:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.309298:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309301:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829437.309304:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829437.309307:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829437.309310:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829437.309314:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829437.309317:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829437.309320:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829437.309325:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309329:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.309332:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309336:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829437.309342:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829437.309347:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309350:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829437.309365:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:3.0:1787829437.309368:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787829437.309370:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
02000000:00000001:3.0:1787829437.309372:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829437.309378:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309382:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829437.309386:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829437.309389:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309391:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829437.309396:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829437.309402:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829437.309404:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829437.309408:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309413:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829437.309414:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829437.309418:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829437.309424:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829437.309429:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829437.309438:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327344256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829437.309464:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829437.309468:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829437.309475:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327344256/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829437.309484:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829437.309488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829437.309493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829437.309496:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829437.309499:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829437.309501:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829437.309506:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3584
00000400:00000200:3.0:1787829437.309510:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829437.309515:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829437.309518:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829437.309523:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309527:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.309531:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309534:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829437.309537:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829437.309540:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829437.309544:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829437.309545:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829437.309548:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829437.309553:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829437.309560:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309563:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829437.309567:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309571:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829437.309575:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829437.309580:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829437.309584:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829437.309587:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829437.309591:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829437.309596:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309599:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829437.309601:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829437.309605:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309609:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829437.309615:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829437.309620:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829437.309626:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829437.309630:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829437.309635:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829437.309639:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829437.309643:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829437.309650:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829437.309685:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829437.309691:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327344384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829437.309702:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829437.309708:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829437.309718:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327344384/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829437.309749:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829437.309756:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3595)
00000800:00000200:0.1:1787829437.309950:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829437.309952:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787829437.310137:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829437.310139:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787829437.310370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829437.310373:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00000040:3.0:1787829438.012703:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298258092, at 4298258112
00000100:00000040:2.0:1787829438.012706:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298258092, at 4298258112
00000100:00000010:3.0:1787829438.012718:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000100:00000010:2.0:1787829438.012721:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000020:00000001:2.0:1787829439.164866:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829439.164870:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829439.164872:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829442.236845:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829442.236850:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829442.236854:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829442.428800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829442.428815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829442.428818:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829442.428824:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829442.428830:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829442.428844:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3590
00000400:00000200:1.0:1787829442.428854:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829442.428861:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829442.428867:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829442.428871:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.428875:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.428878:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.428881:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829442.428886:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829442.428889:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829442.428892:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829442.428896:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829442.428899:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829442.428903:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829442.428906:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.428909:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.428911:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.428913:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829442.428917:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829442.428921:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829442.428923:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829442.428926:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829442.428930:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829442.428932:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.428934:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829442.428937:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829442.428953:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829442.428956:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829442.428961:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829442.428962:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.428965:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829442.428968:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829442.428973:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829442.428981:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829442.428987:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829442.428997:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327344512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829442.429007:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829442.429011:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829442.429015:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327344512/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829442.429033:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829442.429036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829442.429039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829442.429040:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829442.429041:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829442.429043:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829442.429045:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3590
00000400:00000200:1.0:1787829442.429048:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829442.429050:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829442.429052:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829442.429054:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429056:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.429058:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429060:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829442.429061:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829442.429063:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829442.429064:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829442.429066:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829442.429067:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829442.429069:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829442.429072:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429074:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.429075:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429078:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829442.429080:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829442.429082:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429085:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829442.429087:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829442.429094:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829442.429095:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429096:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829442.429101:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829442.429104:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429107:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829442.429114:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829442.429118:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829442.429123:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829442.429126:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429129:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829442.429132:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829442.429135:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829442.429138:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829442.429142:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829442.429149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327344640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829442.429209:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829442.429213:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829442.429221:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327344640/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829442.429240:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829442.429255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829442.429261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829442.429266:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829442.429274:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829442.429283:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829442.429291:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3590
00000400:00000200:1.0:1787829442.429296:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829442.429300:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829442.429303:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829442.429308:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429311:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.429316:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429335:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829442.429345:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829442.429350:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829442.429353:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829442.429356:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829442.429360:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829442.429363:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829442.429370:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429373:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.429376:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429383:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829442.429390:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829442.429400:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429407:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829442.429415:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829442.429423:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829442.429432:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429441:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829442.429452:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829442.429459:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429462:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829442.429467:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829442.429478:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829442.429487:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829442.429493:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429498:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829442.429503:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829442.429507:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829442.429515:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829442.429524:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829442.429535:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327344768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829442.429551:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829442.429555:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829442.429563:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327344768/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829442.429590:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829442.429593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829442.429605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829442.429616:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829442.429619:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829442.429630:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829442.429634:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3590
00000400:00000200:1.0:1787829442.429639:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829442.429646:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829442.429649:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829442.429699:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429701:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.429704:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429706:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829442.429708:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829442.429709:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829442.429711:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829442.429713:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829442.429715:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829442.429716:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829442.429719:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429721:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829442.429722:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429725:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829442.429734:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829442.429738:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829442.429747:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829442.429749:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:1.0:1787829442.429751:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829442.429755:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829442.429758:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829442.429759:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1787829442.429760:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829442.429762:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829442.429771:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429774:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829442.429783:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829442.429790:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829442.429797:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829442.429800:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829442.429807:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829442.429811:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829442.429816:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829442.429825:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829442.429834:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829442.429844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327344896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829442.429877:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829442.429883:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829442.429896:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327344896/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829442.429938:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829442.429946:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3600)
00000800:00000200:0.1:1787829442.430244:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829442.430246:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829442.430459:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829442.430460:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829442.430705:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829442.430708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000040:3.0:1787829443.132724:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298263213, at 4298263232
00000100:00000040:2.0:1787829443.132727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298263213, at 4298263232
00000100:00000010:3.0:1787829443.132750:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000100:00000010:2.0:1787829443.132752:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000020:00000001:2.0:1787829445.308876:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829445.308879:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829445.308882:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829447.548790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829447.548803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829447.548807:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829447.548812:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829447.548818:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829447.548828:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3595
00000400:00000200:1.0:1787829447.548840:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829447.548847:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829447.548852:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829447.548856:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.548860:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.548863:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.548866:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829447.548870:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829447.548873:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829447.548876:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829447.548880:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829447.548883:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829447.548885:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829447.548888:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.548891:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.548893:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.548895:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829447.548899:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829447.548903:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829447.548906:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829447.548911:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829447.548916:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829447.548920:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.548923:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829447.548926:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829447.548938:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829447.548942:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829447.548946:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829447.548949:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.548953:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829447.548956:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829447.548962:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829447.548973:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829447.548981:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829447.548993:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327345024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829447.549006:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829447.549011:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829447.549016:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327345024/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829447.549045:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829447.549053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829447.549057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829447.549058:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829447.549060:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829447.549065:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829447.549069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3595
00000400:00000200:1.0:1787829447.549074:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829447.549079:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829447.549082:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829447.549086:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549092:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.549096:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549100:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829447.549103:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829447.549107:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829447.549109:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829447.549113:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829447.549116:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829447.549120:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829447.549129:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549133:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.549137:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549142:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829447.549147:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829447.549155:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549160:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829447.549166:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829447.549174:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829447.549180:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549184:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829447.549189:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829447.549196:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549199:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829447.549208:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829447.549213:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829447.549216:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829447.549220:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549223:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829447.549227:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829447.549231:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829447.549245:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829447.549252:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829447.549262:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327345152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829447.549279:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829447.549297:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829447.549305:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327345152/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829447.549321:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829447.549326:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829447.549333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829447.549337:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829447.549340:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829447.549346:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829447.549353:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3595
00000400:00000200:1.0:1787829447.549368:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829447.549379:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829447.549383:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829447.549390:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549395:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.549399:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549402:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829447.549406:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829447.549410:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829447.549414:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829447.549418:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829447.549423:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829447.549423:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829447.549429:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829447.549430:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:1.0:1787829447.549436:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549441:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.549447:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549451:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829447.549457:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829447.549464:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549470:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829447.549475:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829447.549480:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829447.549486:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549490:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829447.549494:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829447.549499:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549502:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829447.549509:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829447.549515:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829447.549521:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829447.549526:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549529:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829447.549532:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829447.549535:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829447.549540:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829447.549544:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829447.549554:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327345280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829447.549567:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829447.549572:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829447.549577:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327345280/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829447.549592:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829447.549596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829447.549603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829447.549605:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829447.549607:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829447.549611:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829447.549615:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3595
00000400:00000200:1.0:1787829447.549623:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829447.549630:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829447.549635:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829447.549640:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549644:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.549651:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549686:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829447.549692:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829447.549694:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829447.549698:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829447.549701:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829447.549706:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829447.549711:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829447.549720:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549723:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829447.549726:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549729:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829447.549734:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829447.549741:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829447.549747:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829447.549751:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829447.549756:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829447.549761:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549765:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829447.549770:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829447.549775:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549779:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829447.549785:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829447.549790:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829447.549793:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829447.549796:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829447.549799:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829447.549803:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829447.549805:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829447.549812:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829447.549817:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829447.549828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327345408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829447.549849:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829447.549854:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829447.549864:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327345408/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829447.549885:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829447.549892:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3605)
00000800:00000200:0.1:1787829447.550082:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829447.550085:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829447.550330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829447.550333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829447.550529:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829447.550531:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:3.0:1787829448.252716:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298268332, at 4298268352
00000100:00000010:3.0:1787829448.252729:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000100:00000040:2.0:1787829448.252751:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298268332, at 4298268352
00000100:00000010:2.0:1787829448.252761:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000020:00000001:2.0:1787829448.380900:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829448.380908:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829448.380912:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829451.452866:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829451.452869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829451.452873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829452.668770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829452.668784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829452.668788:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829452.668793:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829452.668799:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829452.668808:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3600
00000400:00000200:3.0:1787829452.668820:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829452.668828:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829452.668832:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829452.668836:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.668841:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.668843:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.668846:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829452.668851:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829452.668853:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829452.668856:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829452.668860:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829452.668863:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829452.668865:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829452.668868:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.668871:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.668873:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.668875:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829452.668878:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829452.668882:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829452.668885:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829452.668889:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829452.668894:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829452.668898:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.668901:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829452.668905:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829452.668917:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829452.668923:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829452.668929:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829452.668932:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.668936:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829452.668948:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829452.668954:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829452.668964:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829452.668972:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829452.668985:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327345536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829452.669005:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829452.669009:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829452.669015:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327345536/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829452.669038:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829452.669042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829452.669046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829452.669048:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829452.669050:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829452.669053:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829452.669056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3600
00000400:00000200:3.0:1787829452.669060:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829452.669065:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829452.669067:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829452.669071:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669074:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.669077:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669079:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829452.669082:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829452.669084:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829452.669087:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829452.669091:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829452.669094:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829452.669098:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829452.669104:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669109:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.669114:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669118:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829452.669125:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829452.669131:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669137:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829452.669143:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829452.669152:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829452.669158:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669161:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829452.669168:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829452.669173:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669179:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829452.669185:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829452.669189:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829452.669194:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829452.669196:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669201:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829452.669203:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829452.669206:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829452.669215:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829452.669222:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829452.669234:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327345664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829452.669264:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829452.669269:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829452.669277:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327345664/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829452.669295:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829452.669303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829452.669310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829452.669315:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829452.669320:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829452.669327:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829452.669336:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3600
00000400:00000200:3.0:1787829452.669357:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829452.669367:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829452.669372:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829452.669380:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669385:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.669392:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829452.669393:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829452.669398:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829452.669399:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000020:00000001:3.0:1787829452.669404:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829452.669409:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829452.669414:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829452.669419:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829452.669423:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829452.669429:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829452.669438:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669443:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.669450:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669455:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829452.669461:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829452.669470:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669476:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829452.669480:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829452.669486:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829452.669492:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669498:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829452.669501:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829452.669509:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669513:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829452.669520:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829452.669525:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829452.669531:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829452.669535:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669540:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829452.669544:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829452.669546:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829452.669555:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829452.669562:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829452.669573:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327345792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829452.669588:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829452.669594:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829452.669600:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327345792/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829452.669626:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829452.669631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829452.669640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829452.669652:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829452.669704:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829452.669709:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829452.669717:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3600
00000400:00000200:3.0:1787829452.669725:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829452.669733:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829452.669738:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829452.669744:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669749:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.669755:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669760:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829452.669765:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829452.669767:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829452.669770:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829452.669772:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829452.669774:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829452.669780:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829452.669789:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669794:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829452.669800:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669805:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829452.669811:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829452.669817:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829452.669822:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829452.669825:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829452.669829:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829452.669833:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669836:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829452.669838:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829452.669842:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669845:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829452.669850:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829452.669855:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829452.669858:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829452.669860:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829452.669864:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829452.669866:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829452.669871:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829452.669875:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829452.669881:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829452.669887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327345920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829452.669900:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829452.669905:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829452.669911:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327345920/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829452.669933:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829452.669940:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3610)
00000800:00000200:0.1:1787829452.670026:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829452.670028:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829452.670272:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829452.670274:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829452.670518:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829452.670520:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787829453.372734:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298273452, at 4298273472
00000100:00000040:3.0:1787829453.372734:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298273452, at 4298273472
00000100:00000010:2.0:1787829453.372753:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000100:00000010:3.0:1787829453.372760:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000020:00000001:2.0:1787829454.524816:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829454.524819:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829454.524823:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829457.596843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829457.596847:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829457.596851:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829457.789009:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829457.789016:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829457.789022:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829457.789034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789053:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829457.789057:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.789064:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829457.789070:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829457.789073:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829457.789078:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327346048:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829457.789084:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829457.789088:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829457.789091:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.789094:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829457.789099:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829457.789104:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829457.789112:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829457.789116:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829457.789123:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829457.789131:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346048, portal 25
00000100:00000001:1.0:1787829457.789136:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829457.789138:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829457.789146:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789159:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829457.789164:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829457.789169:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829457.789172:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327346048, offset 0
00000400:00000200:1.0:1787829457.789180:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829457.789190:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.789198:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829457.789202:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829457.789206:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829457.789208:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829457.789225:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829457.789229:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40475:40475:256:4294967295] 192.168.203.116@tcp LPNI seq info [40475:40475:8:4294967295]
00000400:00000200:1.0:1787829457.789233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.789245:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829457.789255:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.789260:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829457.789266:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829457.789271:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829457.789279:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829457.789284:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829457.789292:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.789295:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829457.789301:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829457.789308:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829457.789313:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829457.789318:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829457.789323:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829457.789330:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789349:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.789357:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789376:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.789380:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829457.789389:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829457.789398:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829457.789404:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298278572
00000100:00000010:1.0:1787829457.789411:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63900.
00000100:00000001:1.0:1787829457.789418:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829457.789423:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298278572
00000100:00000010:1.0:1787829457.789427:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e632a0.
00000100:00000001:1.0:1787829457.789433:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829457.789441:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829457.789444:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.789449:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829457.789455:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457]  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829457.789466:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829457.789469:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829457.789472:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829457.789475:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829457.789478:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829457.789482:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829457.789487:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829457.789505:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789515:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829457.789519:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789522:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.789525:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829457.789528:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789537:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829457.789543:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789553:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829457.789555:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789556:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.789557:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789559:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829457.789562:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829457.789566:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829457.789573:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829457.789578:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829457.789584:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829457.789597:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327346048:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829457.789602:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789607:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.789611:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327346048/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789620:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829457.789622:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829457.789626:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829457.789629:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829457.789631:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829457.789634:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829457.789639:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829457.789642:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829457.789647:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829457.789649:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829457.789652:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829457.789684:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829457.789686:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789688:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789694:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829457.789698:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.789700:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829457.789703:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829457.789705:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829457.789710:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789722:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829457.789724:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.789728:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829457.789731:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829457.789734:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829457.789740:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346176:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829457.789745:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829457.789747:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829457.789750:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.789763:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829457.789767:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829457.789771:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829457.789777:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829457.789782:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829457.789789:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829457.789794:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346176, portal 10
00000100:00000001:1.0:1787829457.789797:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829457.789800:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829457.789811:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789820:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829457.789822:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829457.789824:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829457.789826:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327346176, offset 0
00000400:00000200:1.0:1787829457.789830:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829457.789834:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.789837:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829457.789839:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829457.789841:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829457.789843:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829457.789845:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829457.789847:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40476:40476:256:4294967295] 192.168.203.116@tcp LPNI seq info [40476:40476:8:4294967295]
00000400:00000200:1.0:1787829457.789850:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.789854:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829457.789857:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.789859:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829457.789862:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829457.789864:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829457.789867:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829457.789869:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829457.789871:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.789873:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829457.789875:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829457.789877:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829457.789879:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829457.789881:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829457.789883:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829457.789885:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789890:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.789892:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789901:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.789902:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829457.789904:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829457.789907:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829457.789909:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829457.789911:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298277899
00000100:00000001:1.0:1787829457.789913:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298278572
00000100:00000001:1.0:1787829457.789914:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298277899
00000100:00000001:1.0:1787829457.789915:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298278572
00000100:00000001:1.0:1787829457.789917:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829457.789918:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.789919:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829457.789921:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457]  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829457.789926:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829457.789933:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829457.789935:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829457.789936:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829457.789938:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829457.789939:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829457.789941:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829457.789947:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789951:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829457.789952:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789953:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.789954:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829457.789956:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789961:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829457.789964:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789968:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829457.789970:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789971:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.789971:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.789973:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829457.789974:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829457.789975:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829457.789977:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829457.789979:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829457.789981:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829457.789986:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346176:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829457.789988:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829457.789990:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.789995:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829457.789996:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.789997:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829457.789998:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829457.789999:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829457.790001:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346304:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829457.790003:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829457.790004:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829457.790006:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.790008:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829457.790011:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829457.790013:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829457.790017:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829457.790020:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829457.790024:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829457.790027:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346304, portal 4
00000100:00000001:1.0:1787829457.790029:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829457.790030:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829457.790036:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790043:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829457.790046:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829457.790048:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829457.790051:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327346304, offset 0
00000400:00000200:1.0:1787829457.790055:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829457.790059:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.790063:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829457.790065:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829457.790067:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829457.790068:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829457.790070:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829457.790072:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40477:40477:256:4294967295] 192.168.203.116@tcp LPNI seq info [40477:40477:8:4294967295]
00000400:00000200:1.0:1787829457.790075:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.790080:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829457.790083:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.790085:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829457.790088:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829457.790089:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829457.790093:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829457.790094:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829457.790108:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.790110:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829457.790112:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829457.790114:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829457.790117:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829457.790118:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829457.790120:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829457.790122:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790128:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.790130:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790134:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.790135:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829457.790138:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829457.790140:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829457.790142:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829457.790143:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298277906
00000100:00000001:1.0:1787829457.790145:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298278572
00000100:00000001:1.0:1787829457.790146:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298277906
00000100:00000001:1.0:1787829457.790148:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298278572
00000100:00000001:1.0:1787829457.790149:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829457.790150:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.790151:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829457.790154:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457]  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829457.790159:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829457.790160:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829457.790161:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829457.790163:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829457.790164:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829457.790166:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829457.790168:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829457.790173:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790177:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829457.790178:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790179:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.790180:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829457.790182:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790186:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829457.790188:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790192:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829457.790194:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790195:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.790196:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790197:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829457.790198:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829457.790199:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829457.790202:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829457.790203:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829457.790205:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829457.790229:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346304:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829457.790232:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790233:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.790236:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327346176/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790240:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829457.790241:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829457.790243:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829457.790245:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829457.790246:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829457.790247:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829457.790248:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829457.790250:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829457.790251:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829457.790252:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829457.790253:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829457.790255:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829457.790256:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790257:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.790259:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327346304/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790263:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829457.790264:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829457.790265:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829457.790266:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829457.790267:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829457.790268:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829457.790269:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829457.790269:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829457.790271:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829457.790271:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829457.790272:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829457.790274:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829457.790274:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790275:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790278:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829457.790279:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.790280:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829457.790281:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829457.790282:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829457.790285:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790289:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829457.790290:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.790292:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829457.790293:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829457.790294:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829457.790296:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346432:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829457.790298:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829457.790299:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829457.790300:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829457.790301:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829457.790303:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829457.790304:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829457.790307:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829457.790308:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829457.790310:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829457.790312:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346432, portal 4
00000100:00000001:1.0:1787829457.790314:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829457.790315:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829457.790318:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790323:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829457.790324:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829457.790325:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829457.790327:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327346432, offset 0
00000400:00000200:1.0:1787829457.790330:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829457.790332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.790334:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829457.790336:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829457.790338:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829457.790340:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829457.790341:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829457.790344:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40478:40478:256:4294967295] 192.168.203.116@tcp LPNI seq info [40478:40478:8:4294967295]
00000400:00000200:1.0:1787829457.790346:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829457.790350:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829457.790354:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.790357:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829457.790361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829457.790364:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787829457.790368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829457.790371:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787829457.790375:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829457.790377:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829457.790380:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829457.790383:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829457.790386:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829457.790388:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829457.790391:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829457.790395:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790402:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.790405:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790412:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.790413:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829457.790417:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829457.790420:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829457.790423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829457.790425:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298277911
00000100:00000001:1.0:1787829457.790428:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298278572
00000100:00000001:1.0:1787829457.790430:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298277911
00000100:00000001:1.0:1787829457.790432:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298278572
00000100:00000001:1.0:1787829457.790435:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829457.790437:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.790439:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829457.790442:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457]  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829457.790450:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829457.790452:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829457.790453:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829457.790456:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829457.790458:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829457.790460:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829457.790464:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829457.790472:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790477:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829457.790478:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790479:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.790480:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829457.790482:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790486:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829457.790489:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790493:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829457.790495:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790496:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829457.790496:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790498:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829457.790499:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829457.790501:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829457.790503:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829457.790505:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829457.790507:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829457.790512:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346432:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829457.790514:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790515:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829457.790517:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327346432/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829457.790521:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829457.790522:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829457.790523:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829457.790524:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829457.790525:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829457.790526:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829457.790527:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829457.790528:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829457.790530:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829457.790531:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829457.790532:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829457.790533:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829457.790534:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790536:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829457.790538:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829457.790539:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829457.790540:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829457.790541:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829458.492753:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298278572, at 4298278592
00000100:00000010:3.0:1787829458.492775:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e632a0.
00000100:00000040:2.0:1787829458.492782:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298278572, at 4298278592
00000100:00000010:2.0:1787829458.492799:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63900.
00000020:00000001:2.0:1787829460.668890:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829460.668895:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829460.668899:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829462.908825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829462.908836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829462.908839:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829462.908843:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829462.908848:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829462.908856:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3610
00000400:00000200:3.0:1787829462.908865:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829462.908871:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829462.908874:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829462.908877:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.908879:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.908881:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.908883:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829462.908887:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829462.908889:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829462.908891:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829462.908895:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829462.908896:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829462.908898:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829462.908900:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.908901:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.908903:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.908904:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829462.908907:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829462.908910:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829462.908913:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829462.908915:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829462.908919:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829462.908921:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.908923:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829462.908926:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829462.908935:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829462.908938:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829462.908942:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829462.908943:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.908947:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829462.908951:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829462.908956:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829462.908966:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829462.908975:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829462.908987:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829462.908999:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829462.909004:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829462.909010:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829462.909034:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829462.909038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829462.909042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829462.909043:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829462.909045:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829462.909049:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829462.909052:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3610
00000400:00000200:3.0:1787829462.909056:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829462.909060:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829462.909062:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829462.909065:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909068:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.909070:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909072:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829462.909074:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829462.909076:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829462.909078:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829462.909082:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829462.909084:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829462.909087:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829462.909090:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909092:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.909094:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909096:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829462.909098:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829462.909100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909103:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829462.909105:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829462.909113:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829462.909116:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909119:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829462.909124:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829462.909125:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00000001:3.0:1787829462.909129:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.909133:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829462.909133:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.909138:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:3.0:1787829462.909140:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829462.909145:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000040:1.0:1787829462.909149:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829462.909149:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829462.909153:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909158:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829462.909162:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829462.909165:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.909187:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000010:3.0:1787829462.909188:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:1.0:1787829462.909190:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909195:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.909197:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829462.909203:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829462.909207:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829462.909208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1787829462.909214:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327346560:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829462.909221:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829462.909226:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829462.909230:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909233:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829462.909236:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.909243:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:3.0:1787829462.909244:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787829462.909252:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829462.909257:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909259:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829462.909263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829462.909267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829462.909269:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829462.909271:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829462.909274:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829462.909277:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3610
00000400:00000200:3.0:1787829462.909284:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.909287:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829462.909292:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829462.909295:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829462.909299:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909301:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.909303:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909305:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829462.909308:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829462.909310:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829462.909312:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829462.909314:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829462.909316:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829462.909318:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829462.909322:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909324:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.909326:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829462.909328:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000100:00000001:3.0:1787829462.909330:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829462.909335:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829462.909339:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000010:1.0:1787829462.909340:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000001:3.0:1787829462.909342:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829462.909344:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829462.909347:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829462.909350:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909352:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829462.909355:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829462.909357:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909359:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829462.909364:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829462.909366:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829462.909373:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829462.909374:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909376:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829462.909378:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829462.909380:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829462.909384:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829462.909388:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829462.909391:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346560, portal 25
00000100:00100000:3.0:1787829462.909395:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829462.909396:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829462.909400:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000001:3.0:1787829462.909410:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829462.909413:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829462.909421:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829462.909430:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829462.909431:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829462.909435:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829462.909443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829462.909445:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829462.909447:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829462.909451:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829462.909454:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3610
00000400:00000200:3.0:1787829462.909460:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829462.909467:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1787829462.909468:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829462.909473:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829462.909474:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000040:3.0:1787829462.909478:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909481:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.909492:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909494:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829462.909497:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829462.909500:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829462.909503:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787829462.909504:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000020:00000040:3.0:1787829462.909507:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829462.909511:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829462.909514:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829462.909517:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909519:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829462.909522:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909524:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:1.0:1787829462.909526:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327346560, offset 0
00000020:00000040:3.0:1787829462.909528:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829462.909535:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829462.909539:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829462.909541:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829462.909545:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829462.909548:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829462.909551:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909553:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829462.909556:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829462.909560:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909563:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829462.909568:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000400:00000200:1.0:1787829462.909570:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:3.0:1787829462.909573:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829462.909578:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1787829462.909581:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:3.0:1787829462.909582:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829462.909588:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829462.909590:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829462.909592:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829462.909596:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829462.909600:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829462.909605:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829462.909613:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829462.909615:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829462.909619:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829462.909628:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829462.909633:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3620)
00000400:00000200:1.0:1787829462.909651:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829462.909717:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829462.909721:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829462.909725:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829462.909729:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40479:40479:256:4294967295] 192.168.203.116@tcp LPNI seq info [40479:40479:8:4294967295]
00000400:00000200:1.0:1787829462.909734:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.909745:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829462.909764:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.909769:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829462.909784:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829462.909793:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829462.909799:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829462.909802:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829462.909819:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.909821:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829462.909828:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829462.909832:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829462.909851:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829462.909853:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829462.909858:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829462.909875:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.909888:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.909912:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.909940:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.909943:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829462.909972:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829462.909977:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829462.909981:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298283693
00000100:00000010:1.0:1787829462.910005:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63900.
00000100:00000001:1.0:1787829462.910012:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829462.910032:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298283693
00000100:00000010:1.0:1787829462.910037:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e632a0.
00000100:00000001:1.0:1787829462.910080:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829462.910085:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829462.910087:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.910101:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829462.910114:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462]  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829462.910173:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829462.910236:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829462.910253:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829462.910259:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829462.910275:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829462.910280:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829462.910285:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829462.910310:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.910319:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829462.910322:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.910360:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.910362:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829462.910367:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.910388:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829462.910393:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.910420:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829462.910424:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.910426:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.910427:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.910430:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829462.910444:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829462.910448:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829462.910452:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829462.910523:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829462.910530:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829462.910555:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327346560:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829462.910562:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.910721:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.910727:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327346560/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.910733:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829462.910735:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829462.910738:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829462.910740:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829462.910742:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829462.910743:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829462.910745:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829462.910748:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:1.0:1787829462.910763:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829462.910766:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829462.910769:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:1.0:1787829462.910772:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829462.910774:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.910776:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.910785:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829462.910788:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.910790:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829462.910792:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829462.910794:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829462.910799:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.910807:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829462.910809:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.910812:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829462.910814:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829462.910817:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829462.910821:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346688:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829462.910824:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829462.910827:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829462.910829:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.910831:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829462.910834:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829462.910837:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829462.910841:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829462.910844:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829462.910848:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829462.910852:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346688, portal 10
00000100:00000001:1.0:1787829462.910854:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829462.910856:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:1.0:1787829462.910864:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.910871:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829462.910874:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829462.910876:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829462.910879:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327346688, offset 0
00000400:00000200:1.0:1787829462.910962:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829462.910980:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.910992:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829462.911039:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829462.911044:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829462.911046:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829462.911048:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829462.911051:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40480:40480:256:4294967295] 192.168.203.116@tcp LPNI seq info [40480:40480:8:4294967295]
00000400:00000200:1.0:1787829462.911055:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.911061:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829462.911065:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.911070:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829462.911073:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829462.911076:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829462.911080:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829462.911082:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829462.911085:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.911087:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829462.911089:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829462.911092:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829462.911095:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829462.911096:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829462.911099:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829462.911103:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911110:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.911112:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911118:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911119:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829462.911122:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829462.911125:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829462.911128:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829462.911130:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298283020
00000100:00000001:1.0:1787829462.911132:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298283693
00000100:00000001:1.0:1787829462.911134:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298283020
00000100:00000001:1.0:1787829462.911135:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298283693
00000100:00000001:1.0:1787829462.911137:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829462.911138:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911140:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829462.911142:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462]  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829462.911153:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829462.911154:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829462.911156:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829462.911158:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829462.911159:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829462.911160:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829462.911162:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829462.911168:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911172:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829462.911173:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911187:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.911188:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829462.911192:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911196:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829462.911199:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911204:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829462.911206:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911207:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.911208:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911209:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829462.911211:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829462.911213:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829462.911215:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829462.911216:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829462.911220:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829462.911226:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346688:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829462.911228:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829462.911230:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911234:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829462.911235:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.911237:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829462.911238:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829462.911240:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829462.911242:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346816:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829462.911244:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829462.911245:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829462.911247:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.911248:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829462.911251:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829462.911253:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829462.911256:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829462.911258:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829462.911261:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829462.911263:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346816, portal 4
00000100:00000001:1.0:1787829462.911265:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829462.911266:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:1.0:1787829462.911271:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911275:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829462.911277:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829462.911279:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829462.911281:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327346816, offset 0
00000400:00000200:1.0:1787829462.911284:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829462.911288:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.911291:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829462.911293:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829462.911295:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829462.911297:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829462.911299:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829462.911301:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40481:40481:256:4294967295] 192.168.203.116@tcp LPNI seq info [40481:40481:8:4294967295]
00000400:00000200:1.0:1787829462.911304:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.911309:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829462.911311:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.911313:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829462.911316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829462.911319:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829462.911322:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829462.911324:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829462.911327:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.911328:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829462.911330:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829462.911333:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829462.911335:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829462.911336:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829462.911338:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829462.911341:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911346:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.911348:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911353:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911354:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829462.911356:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829462.911358:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829462.911369:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829462.911371:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298283027
00000100:00000001:1.0:1787829462.911373:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298283693
00000100:00000001:1.0:1787829462.911375:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298283027
00000100:00000001:1.0:1787829462.911379:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298283693
00000100:00000001:1.0:1787829462.911381:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829462.911382:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911384:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829462.911387:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462]  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829462.911392:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829462.911393:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829462.911395:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829462.911396:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829462.911398:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829462.911399:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829462.911401:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829462.911407:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911411:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829462.911412:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911413:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.911414:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829462.911416:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911420:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829462.911423:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911427:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829462.911429:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911430:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.911431:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911433:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829462.911434:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829462.911436:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829462.911438:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829462.911439:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829462.911443:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829462.911451:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346816:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829462.911455:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829462.911458:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911466:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829462.911467:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.911470:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829462.911472:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829462.911474:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829462.911478:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346944:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829462.911481:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829462.911483:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829462.911485:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829462.911487:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829462.911491:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829462.911493:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829462.911497:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829462.911499:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829462.911502:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829462.911505:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327346944, portal 4
00000100:00000001:1.0:1787829462.911507:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829462.911508:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:1.0:1787829462.911513:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911518:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829462.911520:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829462.911521:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829462.911523:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327346944, offset 0
00000400:00000200:1.0:1787829462.911526:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829462.911529:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.911531:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829462.911534:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829462.911536:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829462.911538:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829462.911539:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829462.911542:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40482:40482:256:4294967295] 192.168.203.116@tcp LPNI seq info [40482:40482:8:4294967295]
00000400:00000200:1.0:1787829462.911544:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829462.911549:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829462.911552:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.911554:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829462.911557:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829462.911559:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829462.911562:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829462.911564:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829462.911567:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829462.911569:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829462.911571:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829462.911573:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829462.911575:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829462.911577:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829462.911578:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829462.911581:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911591:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.911593:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911597:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911598:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829462.911600:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829462.911602:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829462.911604:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829462.911606:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298283032
00000100:00000001:1.0:1787829462.911607:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298283693
00000100:00000001:1.0:1787829462.911609:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298283032
00000100:00000001:1.0:1787829462.911610:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298283693
00000100:00000001:1.0:1787829462.911611:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829462.911613:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911614:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829462.911616:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462]  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829462.911621:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829462.911622:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829462.911623:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829462.911625:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829462.911626:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829462.911628:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829462.911629:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829462.911635:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911639:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829462.911641:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911642:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.911643:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829462.911646:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911650:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829462.911652:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911672:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829462.911674:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911675:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829462.911676:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911677:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829462.911679:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829462.911681:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829462.911683:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829462.911685:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829462.911688:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829462.911696:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327346944:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829462.911700:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911703:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.911706:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327346688/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911714:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829462.911716:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829462.911719:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829462.911722:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829462.911724:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829462.911726:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829462.911728:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829462.911730:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:1.0:1787829462.911733:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829462.911734:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829462.911736:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:1.0:1787829462.911739:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829462.911741:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911743:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.911746:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327346816/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911751:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829462.911752:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829462.911753:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829462.911755:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829462.911756:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829462.911756:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829462.911758:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829462.911758:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:1.0:1787829462.911760:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829462.911760:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829462.911761:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:1.0:1787829462.911763:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829462.911763:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911764:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829462.911766:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327346944/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829462.911771:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829462.911772:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829462.911773:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829462.911774:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829462.911775:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829462.911775:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829462.911776:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829462.911777:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:1.0:1787829462.911778:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829462.911779:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829462.911780:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:1.0:1787829462.911782:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829462.911783:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911784:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829462.911787:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829462.911788:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829462.911790:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829462.911791:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829463.612787:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298283693, at 4298283712
00000100:00000040:3.0:1787829463.612787:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298283693, at 4298283712
00000100:00000010:2.0:1787829463.612805:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63900.
00000100:00000010:3.0:1787829463.612807:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e632a0.
00000020:00000001:2.0:1787829463.740916:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829463.740922:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829463.740926:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829466.812882:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829466.812887:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829466.812891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829467.516813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829467.516822:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829467.516824:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829467.516829:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829467.516833:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829467.516837:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829467.516839:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829467.516842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829467.516844:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829468.028772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829468.028785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829468.028789:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829468.028794:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829468.028801:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829468.028810:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3615
00000400:00000200:3.0:1787829468.028822:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829468.028830:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829468.028834:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829468.028838:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.028842:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829468.028845:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.028848:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829468.028854:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829468.028857:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829468.028860:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829468.028864:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829468.028867:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829468.028870:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829468.028874:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.028876:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829468.028879:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.028881:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829468.028885:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829468.028889:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829468.028892:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829468.028896:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829468.028900:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829468.028904:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.028906:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829468.028910:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829468.028919:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829468.028924:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829468.028930:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829468.028933:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.028937:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829468.028941:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829468.028947:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829468.028957:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829468.028965:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829468.028976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829468.028989:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829468.028995:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829468.029000:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829468.029029:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829468.029033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829468.029037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829468.029039:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829468.029041:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829468.029044:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829468.029048:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3615
00000400:00000200:3.0:1787829468.029053:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829468.029057:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829468.029060:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829468.029065:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029067:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829468.029069:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787829468.029070:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829468.029076:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829468.029076:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829468.029080:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787829468.029081:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829468.029085:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829468.029089:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787829468.029090:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787829468.029092:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829468.029098:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829468.029105:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:3.0:1787829468.029106:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829468.029108:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.029113:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787829468.029114:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:1.0:1787829468.029116:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829468.029120:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787829468.029121:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829468.029126:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00100000:1.0:1787829468.029127:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327347072:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829468.029132:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829468.029133:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1787829468.029138:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:1.0:1787829468.029149:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829468.029154:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829468.029157:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829468.029158:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829468.029163:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829468.029165:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:3.0:1787829468.029167:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829468.029168:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.029176:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829468.029176:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1787829468.029182:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
02000000:00000001:3.0:1787829468.029182:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029187:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787829468.029193:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000008:00100000:3.0:1787829468.029195:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829468.029200:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029202:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829468.029203:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347072, portal 25
00000100:00000010:3.0:1787829468.029208:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000100:00000001:1.0:1787829468.029209:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829468.029213:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000020:00000040:3.0:1787829468.029214:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000040:1.0:1787829468.029226:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829468.029228:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829468.029232:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029236:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829468.029239:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829468.029243:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029244:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829468.029250:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000010:3.0:1787829468.029250:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000400:00000010:1.0:1787829468.029254:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000001:3.0:1787829468.029256:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829468.029260:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327347072, offset 0
00000100:00100000:3.0:1787829468.029266:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829468.029270:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029280:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829468.029281:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829468.029287:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.029291:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:3.0:1787829468.029297:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829468.029298:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829468.029306:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829468.029313:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829468.029316:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029318:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.029322:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40483:40483:256:4294967295] 192.168.203.116@tcp LPNI seq info [40483:40483:8:4294967295]
00000100:00080000:3.0:1787829468.029324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829468.029327:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829468.029333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829468.029336:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787829468.029340:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1787829468.029340:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829468.029350:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1787829468.029353:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829468.029357:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3615
00000800:00000010:1.0:1787829468.029358:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000400:00000200:3.0:1787829468.029366:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:1.0:1787829468.029370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829468.029375:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000400:00000200:3.0:1787829468.029375:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829468.029382:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:1.1:1787829468.029384:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829468.029389:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000040:3.0:1787829468.029390:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000100:1.0:1787829468.029396:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029397:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:1.0:1787829468.029399:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000100:00000040:3.0:1787829468.029404:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829468.029406:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787829468.029409:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829468.029414:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1787829468.029415:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829468.029418:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:1.0:1787829468.029420:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000020:00000001:3.0:1787829468.029421:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787829468.029423:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000020:00000040:3.0:1787829468.029426:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000001:1.0:1787829468.029428:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:3.0:1787829468.029432:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:1.0:1787829468.029434:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829468.029438:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829468.029448:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829468.029451:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787829468.029452:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829468.029456:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829468.029460:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029467:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829468.029469:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029472:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:3.0:1787829468.029472:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829468.029479:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829468.029480:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029484:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.029489:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829468.029491:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829468.029492:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298288812
02000000:00000001:3.0:1787829468.029496:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:1.0:1787829468.029499:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e633c0.
02000000:00000001:3.0:1787829468.029501:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029505:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829468.029508:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000008:00100000:3.0:1787829468.029509:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829468.029515:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029516:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298288812
00000100:00000001:3.0:1787829468.029517:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829468.029522:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63600.
00000100:00000001:1.0:1787829468.029526:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000010:3.0:1787829468.029527:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000100:00000001:1.0:1787829468.029529:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1787829468.029533:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:1.0:1787829468.029534:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029540:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1787829468.029541:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829468.029545:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829468.029550:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829468/real 1787829468]  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829468.029551:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829468.029554:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829468.029558:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829468.029567:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000400:00000001:1.0:1787829468.029569:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829468.029573:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787829468.029574:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829468.029576:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000100:00100000:3.0:1787829468.029583:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829468.029591:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000100:00000001:3.0:1787829468.029595:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787829468.029596:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:3.0:1787829468.029601:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029602:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1787829468.029608:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829468.029611:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829468.029630:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829468.029636:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829468.029637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829468.029647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829468.029651:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787829468.029651:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829468.029678:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829468.029687:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829468.029692:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3615
00000400:00000200:3.0:1787829468.029697:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829468.029701:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787829468.029702:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829468.029707:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:3.0:1787829468.029708:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829468.029710:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829468.029717:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029721:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1787829468.029723:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829468.029728:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829468.029733:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829468.029736:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1787829468.029738:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829468.029741:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000040:1.0:1787829468.029742:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829468.029745:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829468.029749:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829468.029752:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:3.0:1787829468.029754:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:1.0:1787829468.029756:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.029761:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:3.0:1787829468.029761:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:1.0:1787829468.029762:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.029769:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:3.0:1787829468.029770:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829468.029772:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787829468.029776:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:1.0:1787829468.029777:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829468.029784:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000040:3.0:1787829468.029784:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829468.029788:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1787829468.029790:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:1.0:1787829468.029793:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787829468.029798:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00100000:1.0:1787829468.029804:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327347072:192.168.203.116@tcp:250:kworker.0
00000100:00080000:3.0:1787829468.029807:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829468.029810:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.029815:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787829468.029815:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829468.029820:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787829468.029821:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327347072/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829468.029827:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829468.029828:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829468.029831:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:3.0:1787829468.029833:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829468.029834:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829468.029839:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1787829468.029839:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000020:00000001:1.0:1787829468.029844:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000008:00100000:3.0:1787829468.029845:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000020:00000040:1.0:1787829468.029846:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829468.029850:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
00000008:00000001:3.0:1787829468.029852:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029855:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829468.029856:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
00000100:00000010:3.0:1787829468.029860:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
02000000:00000001:1.0:1787829468.029862:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829468.029866:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000040:3.0:1787829468.029867:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000010:1.0:1787829468.029868:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:1.0:1787829468.029875:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
02000000:00000001:3.0:1787829468.029875:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829468.029878:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787829468.029879:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029880:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829468.029886:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829468.029889:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787829468.029890:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:3.0:1787829468.029892:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029893:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029897:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829468.029899:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000010:3.0:1787829468.029900:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:1.0:1787829468.029901:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829468.029906:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829468.029908:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.029916:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829468.029918:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.029921:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:3.0:1787829468.029921:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787829468.029923:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829468.029933:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829468.029934:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829468.029940:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829468.029942:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347200:192.168.203.116@tcp:38:kworker.0
00000100:00000040:3.0:1787829468.029949:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.029950:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829468.029961:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787829468.029962:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.029965:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829468.029970:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3625)
02000000:00000001:1.0:1787829468.029974:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829468.029979:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829468.029987:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.029992:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829468.029995:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829468.029999:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829468.030002:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347200, portal 10
00000100:00000001:1.0:1787829468.030003:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829468.030004:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:1.0:1787829468.030011:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030016:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829468.030018:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829468.030019:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829468.030021:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327347200, offset 0
00000400:00000200:1.0:1787829468.030024:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829468.030027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829468.030031:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829468.030033:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829468.030035:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829468.030037:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829468.030039:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829468.030041:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40484:40484:256:4294967295] 192.168.203.116@tcp LPNI seq info [40484:40484:8:4294967295]
00000400:00000200:1.0:1787829468.030044:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829468.030049:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829468.030052:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829468.030054:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829468.030057:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829468.030059:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829468.030063:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829468.030065:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829468.030068:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829468.030069:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829468.030071:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829468.030074:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829468.030076:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829468.030078:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829468.030079:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829468.030082:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030088:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829468.030090:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030095:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030096:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829468.030098:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829468.030100:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829468.030102:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829468.030104:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298288139
00000100:00000001:1.0:1787829468.030106:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298288812
00000100:00000001:1.0:1787829468.030108:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298288139
00000100:00000001:1.0:1787829468.030110:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298288812
00000100:00000001:1.0:1787829468.030111:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829468.030113:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030114:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829468.030116:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829468/real 1787829468]  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829468.030121:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829468.030122:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829468.030123:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829468.030125:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829468.030126:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829468.030128:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829468.030130:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829468.030135:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030152:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829468.030153:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030155:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829468.030156:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829468.030159:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030163:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829468.030166:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030171:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829468.030172:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030174:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829468.030175:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030176:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829468.030178:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829468.030180:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829468.030182:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829468.030184:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829468.030186:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829468.030191:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347200:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829468.030195:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829468.030199:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030206:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829468.030207:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.030210:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829468.030212:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829468.030214:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829468.030217:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347328:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829468.030221:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829468.030223:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829468.030225:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.030227:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829468.030231:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829468.030233:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.030238:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829468.030241:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829468.030244:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829468.030248:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347328, portal 4
00000100:00000001:1.0:1787829468.030249:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829468.030250:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:1.0:1787829468.030256:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030260:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829468.030262:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829468.030264:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829468.030266:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327347328, offset 0
00000400:00000200:1.0:1787829468.030268:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829468.030271:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829468.030274:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829468.030276:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829468.030277:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829468.030279:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829468.030281:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829468.030283:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40485:40485:256:4294967295] 192.168.203.116@tcp LPNI seq info [40485:40485:8:4294967295]
00000400:00000200:1.0:1787829468.030286:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829468.030291:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829468.030294:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829468.030296:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829468.030299:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829468.030301:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829468.030305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829468.030307:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829468.030309:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829468.030311:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829468.030313:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829468.030315:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829468.030318:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829468.030319:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829468.030321:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829468.030323:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030329:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829468.030331:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030336:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030337:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829468.030339:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829468.030341:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829468.030344:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829468.030345:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298288146
00000100:00000001:1.0:1787829468.030347:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298288812
00000100:00000001:1.0:1787829468.030349:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298288146
00000100:00000001:1.0:1787829468.030351:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298288812
00000100:00000001:1.0:1787829468.030352:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829468.030353:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030355:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829468.030357:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829468/real 1787829468]  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829468.030362:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829468.030363:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829468.030364:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829468.030366:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829468.030367:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829468.030368:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829468.030370:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829468.030376:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030380:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829468.030381:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030382:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829468.030383:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829468.030386:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030391:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829468.030393:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030397:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829468.030399:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030400:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829468.030401:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030402:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829468.030403:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829468.030404:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829468.030406:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829468.030407:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829468.030410:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829468.030414:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347328:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829468.030417:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030418:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829468.030420:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327347200/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030424:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829468.030425:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829468.030426:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829468.030428:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829468.030429:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829468.030430:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829468.030432:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829468.030433:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:1.0:1787829468.030435:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829468.030436:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829468.030438:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:1.0:1787829468.030440:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829468.030441:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030442:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829468.030444:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327347328/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030448:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829468.030449:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829468.030449:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829468.030451:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829468.030451:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829468.030452:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829468.030453:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829468.030454:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:1.0:1787829468.030455:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829468.030456:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829468.030457:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:1.0:1787829468.030458:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829468.030459:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030460:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030462:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829468.030463:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030464:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829468.030465:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829468.030466:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829468.030468:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030472:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829468.030473:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.030474:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829468.030475:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829468.030476:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829468.030478:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347456:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829468.030481:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829468.030482:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829468.030483:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829468.030485:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829468.030487:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829468.030488:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829468.030490:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829468.030492:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829468.030494:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829468.030496:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347456, portal 4
00000100:00000001:1.0:1787829468.030497:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829468.030498:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:1.0:1787829468.030502:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030507:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829468.030508:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829468.030510:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829468.030512:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327347456, offset 0
00000400:00000200:1.0:1787829468.030514:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829468.030516:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829468.030518:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829468.030520:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829468.030522:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829468.030524:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829468.030526:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829468.030528:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40486:40486:256:4294967295] 192.168.203.116@tcp LPNI seq info [40486:40486:8:4294967295]
00000400:00000200:1.0:1787829468.030531:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829468.030536:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829468.030540:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829468.030544:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829468.030548:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829468.030552:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829468.030556:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829468.030559:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829468.030563:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829468.030565:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829468.030568:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829468.030572:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829468.030575:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829468.030577:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829468.030580:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829468.030584:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030593:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829468.030596:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030604:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030605:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829468.030609:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829468.030612:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829468.030616:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829468.030619:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298288151
00000100:00000001:1.0:1787829468.030622:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298288812
00000100:00000001:1.0:1787829468.030624:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298288151
00000100:00000001:1.0:1787829468.030627:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298288812
00000100:00000001:1.0:1787829468.030629:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829468.030632:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030634:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829468.030637:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829468/real 1787829468]  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829468.030643:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829468.030644:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829468.030646:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829468.030648:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829468.030649:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829468.030651:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829468.030653:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829468.030675:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030680:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829468.030681:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030682:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829468.030683:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829468.030685:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030692:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829468.030696:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030704:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829468.030716:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030718:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829468.030719:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030724:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829468.030726:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829468.030729:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829468.030733:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829468.030735:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829468.030739:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829468.030747:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347456:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829468.030752:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030754:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829468.030757:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327347456/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829468.030764:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829468.030765:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829468.030767:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829468.030770:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829468.030772:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829468.030773:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829468.030775:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829468.030777:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:1.0:1787829468.030779:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829468.030780:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829468.030781:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:1.0:1787829468.030784:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829468.030785:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030786:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829468.030790:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829468.030791:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829468.030792:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829468.030794:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829468.540766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829468.540775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829468.540778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829468.540786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829468.540790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829468.540795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829468.540799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829468.540803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829468.540807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829468.732747:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298288812, at 4298288832
00000100:00000040:3.0:1787829468.732747:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298288812, at 4298288832
00000100:00000010:2.0:1787829468.732770:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e633c0.
00000100:00000010:3.0:1787829468.732777:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63600.
00000100:00000001:2.0:1787829469.564777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829469.564785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829469.564787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829469.564792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829469.564796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829469.564800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829469.564802:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829469.564805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829469.564807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829469.884836:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829469.884841:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829469.884846:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829470.588763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829470.588770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829470.588777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829470.588784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829470.588787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829470.588791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829470.588794:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829470.588796:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829470.588799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829471.612785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829471.612795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829471.612798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829471.612804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829471.612808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829471.612812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829471.612814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829471.612818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829471.612820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829472.636766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829472.636775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829472.636777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829472.636782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829472.636786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829472.636790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829472.636793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829472.636795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829472.636804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829472.956879:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829472.956884:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829472.956887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829473.148772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829473.148787:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829473.148790:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829473.148804:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829473.148810:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829473.148820:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3620
00000400:00000200:3.0:1787829473.148832:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829473.148840:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829473.148844:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829473.148848:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.148852:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829473.148855:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.148858:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829473.148863:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829473.148865:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829473.148869:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829473.148874:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829473.148876:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829473.148879:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829473.148883:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.148885:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829473.148888:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.148890:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829473.148894:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829473.148898:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829473.148901:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829473.148905:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829473.148909:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829473.148913:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.148915:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829473.148919:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829473.148931:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829473.148937:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829473.148942:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829473.148944:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.148949:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829473.148953:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829473.148958:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829473.148969:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829473.148978:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829473.148989:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829473.149001:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829473.149007:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829473.149013:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829473.149041:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829473.149044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829473.149046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829473.149048:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829473.149049:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829473.149052:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829473.149054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3620
00000400:00000200:3.0:1787829473.149057:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829473.149059:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829473.149060:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829473.149063:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149065:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829473.149067:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149069:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829473.149070:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829473.149071:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829473.149073:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829473.149076:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829473.149078:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829473.149080:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829473.149082:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787829473.149083:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149087:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829473.149088:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829473.149089:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829473.149090:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829473.149093:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787829473.149094:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829473.149098:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149101:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829473.149116:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829473.149117:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829473.149124:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829473.149129:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787829473.149129:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149131:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829473.149133:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787829473.149137:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829473.149138:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1787829473.149141:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.149142:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829473.149144:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.149150:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829473.149151:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829473.149156:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829473.149158:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829473.149159:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149160:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829473.149163:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829473.149165:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829473.149167:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327347584:192.168.203.116@tcp:250:kworker.0
02000000:00000010:3.0:1787829473.149171:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:1.0:1787829473.149176:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829473.149177:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.149184:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:3.0:1787829473.149185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787829473.149190:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.149198:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829473.149202:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829473.149206:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829473.149207:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829473.149211:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829473.149214:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829473.149221:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829473.149227:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829473.149235:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829473.149246:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347584, portal 25
00000100:00000001:3.0:1787829473.149246:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829473.149251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829473.149252:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829473.149254:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00080000:3.0:1787829473.149262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:1.0:1787829473.149264:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829473.149266:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829473.149271:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829473.149278:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787829473.149280:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1787829473.149286:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3620
00000100:00000040:1.0:1787829473.149287:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829473.149291:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829473.149296:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327347584, offset 0
00000400:00000200:3.0:1787829473.149298:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.149305:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787829473.149306:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829473.149311:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787829473.149316:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829473.149316:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149321:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829473.149325:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149329:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829473.149332:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787829473.149334:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000040:3.0:1787829473.149335:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829473.149340:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787829473.149341:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1787829473.149345:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:1.0:1787829473.149348:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:3.0:1787829473.149351:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829473.149354:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000001:3.0:1787829473.149358:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829473.149359:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829473.149368:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40487:40487:256:4294967295] 192.168.203.116@tcp LPNI seq info [40487:40487:8:4294967295]
00000100:00000040:3.0:1787829473.149368:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149376:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829473.149377:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829473.149383:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149390:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829473.149391:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1787829473.149397:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:1.0:1787829473.149402:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829473.149405:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000010:1.0:1787829473.149408:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000100:00000001:3.0:1787829473.149409:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829473.149415:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:1.0:1787829473.149419:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:3.0:1787829473.149419:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829473.149424:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829473.149426:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:3.0:1787829473.149427:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829473.149431:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.1:1787829473.149436:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:3.0:1787829473.149437:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149441:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829473.149443:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000010:3.0:1787829473.149446:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000800:00000100:1.0:1787829473.149451:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000040:3.0:1787829473.149452:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000010:1.0:1787829473.149458:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
02000000:00000001:3.0:1787829473.149459:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829473.149462:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149466:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829473.149468:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787829473.149471:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1787829473.149471:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829473.149481:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000400:00000200:1.0:1787829473.149483:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787829473.149487:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829473.149492:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829473.149497:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829473.149503:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:3.0:1787829473.149505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787829473.149511:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829473.149524:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829473.149529:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.149536:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787829473.149538:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829473.149545:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829473.149557:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.149560:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.149563:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:3.0:1787829473.149563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829473.149570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829473.149572:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149573:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829473.149577:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829473.149580:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829473.149583:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787829473.149585:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298293932
00000100:00080000:3.0:1787829473.149589:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3620
00000100:00000010:1.0:1787829473.149594:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63660.
00000400:00000200:3.0:1787829473.149599:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829473.149604:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000400:00000200:3.0:1787829473.149605:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829473.149610:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829473.149612:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298293932
00000100:00000010:1.0:1787829473.149616:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63a80.
00000100:00000001:1.0:1787829473.149618:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000040:3.0:1787829473.149620:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829473.149622:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829473.149623:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829473.149628:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829473.149629:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149633:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829473.149636:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:3.0:1787829473.149637:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829473.149640:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829473.149642:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829473.149644:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829473.149647:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00100000:1.0:1787829473.149649:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829473/real 1787829473]  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829473.149649:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829473.149676:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149677:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829473.149679:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149680:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829473.149682:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829473.149685:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829473.149687:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829473.149689:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829473.149691:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829473.149696:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149700:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829473.149704:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000001:1.0:1787829473.149707:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00000001:3.0:1787829473.149707:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149709:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787829473.149713:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000010:3.0:1787829473.149714:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000400:00000010:1.0:1787829473.149718:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000020:00000040:3.0:1787829473.149718:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829473.149725:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1787829473.149727:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
02000000:00000001:3.0:1787829473.149727:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829473.149728:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829473.149731:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1787829473.149733:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
02000000:00000001:3.0:1787829473.149734:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.149741:0:2346:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:3.0:1787829473.149742:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829473.149747:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829473.149751:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787829473.149755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829473.149780:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829473.149782:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829473.149785:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829473.149788:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829473.149815:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829473.149820:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3630)
00000100:00000001:1.0:1787829473.149828:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829473.149832:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.149837:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.149840:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829473.149847:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.149856:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829473.149862:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.149871:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829473.149873:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.149875:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.149876:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.149878:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829473.149880:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829473.149882:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829473.149885:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829473.149887:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829473.149890:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829473.149904:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327347584:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829473.149909:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.149910:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.149913:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327347584/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.149922:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829473.149925:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829473.149927:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829473.149930:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829473.149931:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829473.149933:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829473.149934:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829473.149937:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:1.0:1787829473.149941:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829473.149942:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829473.149944:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:1.0:1787829473.149949:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829473.149950:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.149952:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.149959:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829473.149961:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.149963:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829473.149964:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829473.149965:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829473.149968:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.149972:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829473.149974:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.149975:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829473.149976:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829473.149978:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829473.149980:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347712:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829473.149983:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829473.149984:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829473.149986:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.149987:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829473.149990:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829473.149991:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829473.149994:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829473.149997:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829473.150000:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829473.150003:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347712, portal 10
00000100:00000001:1.0:1787829473.150004:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829473.150006:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:1.0:1787829473.150011:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150016:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829473.150018:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829473.150019:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829473.150021:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327347712, offset 0
00000400:00000200:1.0:1787829473.150024:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829473.150027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.150030:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829473.150033:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829473.150034:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829473.150036:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829473.150038:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829473.150040:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40488:40488:256:4294967295] 192.168.203.116@tcp LPNI seq info [40488:40488:8:4294967295]
00000400:00000200:1.0:1787829473.150043:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.150048:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829473.150051:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829473.150054:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829473.150057:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829473.150059:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829473.150062:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829473.150064:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829473.150067:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829473.150069:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829473.150071:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829473.150074:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829473.150076:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829473.150077:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829473.150079:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829473.150082:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150087:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.150089:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150094:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150095:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829473.150098:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829473.150100:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829473.150102:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829473.150117:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298293259
00000100:00000001:1.0:1787829473.150120:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298293932
00000100:00000001:1.0:1787829473.150122:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298293259
00000100:00000001:1.0:1787829473.150123:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298293932
00000100:00000001:1.0:1787829473.150125:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829473.150126:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150128:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829473.150131:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829473/real 1787829473]  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829473.150136:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829473.150137:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829473.150139:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829473.150141:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829473.150142:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829473.150144:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829473.150146:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829473.150155:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150169:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829473.150170:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150172:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.150174:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829473.150178:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150186:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829473.150190:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150198:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829473.150201:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150202:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.150203:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150205:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829473.150207:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829473.150209:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829473.150211:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829473.150213:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829473.150216:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829473.150221:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347712:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829473.150224:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829473.150226:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150231:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829473.150232:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.150233:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829473.150235:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829473.150237:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829473.150238:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829473.150241:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829473.150242:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829473.150243:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.150244:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829473.150247:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829473.150248:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829473.150251:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829473.150253:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829473.150256:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829473.150258:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347840, portal 4
00000100:00000001:1.0:1787829473.150260:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829473.150261:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:1.0:1787829473.150265:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150270:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829473.150272:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829473.150273:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829473.150275:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327347840, offset 0
00000400:00000200:1.0:1787829473.150277:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829473.150280:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.150282:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829473.150285:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829473.150286:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829473.150288:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829473.150290:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829473.150292:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40489:40489:256:4294967295] 192.168.203.116@tcp LPNI seq info [40489:40489:8:4294967295]
00000400:00000200:1.0:1787829473.150295:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.150300:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829473.150303:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829473.150305:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829473.150308:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829473.150310:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829473.150313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829473.150315:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829473.150333:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829473.150335:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829473.150338:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829473.150340:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829473.150342:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829473.150344:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829473.150346:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829473.150348:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150354:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.150356:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150360:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150362:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829473.150364:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829473.150367:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829473.150369:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829473.150371:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298293266
00000100:00000001:1.0:1787829473.150373:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298293932
00000100:00000001:1.0:1787829473.150375:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298293266
00000100:00000001:1.0:1787829473.150376:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298293932
00000100:00000001:1.0:1787829473.150377:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829473.150378:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150380:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829473.150382:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829473/real 1787829473]  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829473.150388:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829473.150389:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829473.150390:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829473.150392:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829473.150393:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829473.150394:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829473.150396:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829473.150402:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150406:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829473.150407:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150408:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.150409:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829473.150411:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150415:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829473.150418:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150422:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829473.150424:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150425:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.150426:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150428:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829473.150430:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829473.150432:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829473.150435:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829473.150437:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829473.150442:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829473.150450:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829473.150454:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829473.150457:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150464:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829473.150465:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.150467:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829473.150470:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829473.150472:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829473.150475:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829473.150478:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829473.150480:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829473.150482:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829473.150484:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829473.150487:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829473.150488:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829473.150491:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829473.150494:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829473.150497:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829473.150500:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327347968, portal 4
00000100:00000001:1.0:1787829473.150502:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829473.150504:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:1.0:1787829473.150510:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150517:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829473.150519:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829473.150522:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829473.150525:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327347968, offset 0
00000400:00000200:1.0:1787829473.150529:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829473.150533:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.150537:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829473.150539:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829473.150542:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829473.150544:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829473.150547:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829473.150549:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40490:40490:256:4294967295] 192.168.203.116@tcp LPNI seq info [40490:40490:8:4294967295]
00000400:00000200:1.0:1787829473.150553:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829473.150559:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829473.150563:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829473.150566:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829473.150570:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829473.150572:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829473.150576:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829473.150579:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829473.150582:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829473.150584:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829473.150587:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829473.150590:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829473.150593:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829473.150595:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829473.150597:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829473.150600:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150608:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.150610:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150617:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150619:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829473.150623:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829473.150625:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829473.150629:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829473.150631:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298293271
00000100:00000001:1.0:1787829473.150634:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298293932
00000100:00000001:1.0:1787829473.150636:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298293271
00000100:00000001:1.0:1787829473.150638:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298293932
00000100:00000001:1.0:1787829473.150640:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829473.150642:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150644:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829473.150648:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829473/real 1787829473]  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829473.150674:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829473.150675:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829473.150677:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829473.150679:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829473.150680:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829473.150682:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829473.150685:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829473.150691:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150695:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829473.150696:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150697:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.150698:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829473.150700:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150705:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829473.150709:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150716:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829473.150720:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150721:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829473.150722:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150724:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829473.150726:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829473.150729:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829473.150732:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829473.150735:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829473.150739:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829473.150749:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327347968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829473.150753:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150755:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.150759:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327347712/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150765:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829473.150766:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829473.150768:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829473.150770:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829473.150771:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829473.150772:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829473.150774:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829473.150775:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:1.0:1787829473.150777:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829473.150778:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829473.150779:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:1.0:1787829473.150781:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829473.150782:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150784:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.150788:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327347840/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150801:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829473.150802:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829473.150804:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829473.150807:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829473.150808:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829473.150809:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829473.150812:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829473.150814:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:1.0:1787829473.150816:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829473.150817:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829473.150819:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:1.0:1787829473.150823:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829473.150825:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150827:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829473.150831:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327347968/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829473.150838:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829473.150839:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829473.150840:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829473.150842:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829473.150843:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829473.150844:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829473.150846:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829473.150847:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:1.0:1787829473.150848:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829473.150849:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829473.150850:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829473.150852:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829473.150852:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150854:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829473.150856:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829473.150857:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829473.150858:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829473.150860:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829473.660815:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829473.660822:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829473.660825:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829473.660829:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829473.660833:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829473.660838:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829473.660844:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829473.660846:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829473.660849:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829473.852795:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298293932, at 4298293952
00000100:00000040:3.0:1787829473.852795:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298293932, at 4298293952
00000100:00000010:2.0:1787829473.852819:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63660.
00000100:00000010:3.0:1787829473.852821:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a80.
00000100:00000001:2.0:1787829474.684764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829474.684775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829474.684778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829474.684786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829474.684790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829474.684794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829474.684797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829474.684799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829474.684803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829475.708772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829475.708778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829475.708779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829475.708783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829475.708787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829475.708790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829475.708792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829475.708794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829475.708796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829476.028854:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829476.028861:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829476.028866:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829476.732778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829476.732784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829476.732787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829476.732792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829476.732796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829476.732799:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829476.732803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829476.732805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829476.732809:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829477.756762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829477.756770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829477.756773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829477.756780:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829477.756784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829477.756787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829477.756790:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829477.756794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829477.756797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.269004:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829478.269011:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829478.269014:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829478.269024:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269034:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829478.269037:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.269044:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829478.269046:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829478.269050:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829478.269056:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327348096:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829478.269062:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829478.269079:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829478.269083:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.269087:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829478.269094:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829478.269097:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829478.269104:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829478.269109:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829478.269118:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829478.269127:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348096, portal 25
00000100:00000001:1.0:1787829478.269133:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829478.269136:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829478.269147:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269165:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829478.269170:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829478.269173:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829478.269178:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327348096, offset 0
00000400:00000200:1.0:1787829478.269188:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829478.269202:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.269210:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829478.269217:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829478.269224:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829478.269230:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829478.269235:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829478.269240:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40491:40491:256:4294967295] 192.168.203.116@tcp LPNI seq info [40491:40491:8:4294967295]
00000400:00000200:1.0:1787829478.269260:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.269279:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829478.269291:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.269300:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829478.269311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829478.269318:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787829478.269330:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829478.269338:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787829478.269348:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.269352:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829478.269360:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829478.269367:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829478.269373:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829478.269376:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829478.269382:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829478.269388:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269400:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.269406:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269416:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.269420:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829478.269427:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829478.269433:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829478.269436:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298299052
00000100:00000010:1.0:1787829478.269442:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e636c0.
00000100:00000001:1.0:1787829478.269451:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829478.269457:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298299052
00000100:00000010:1.0:1787829478.269461:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63c60.
00000100:00000001:1.0:1787829478.269471:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829478.269482:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829478.269487:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.269493:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829478.269499:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829478/real 1787829478]  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829478.269510:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829478.269514:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829478.269519:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829478.269525:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829478.269529:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829478.269535:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829478.269546:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829478.269570:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269580:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829478.269583:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269585:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.269587:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829478.269594:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269603:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829478.269609:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269615:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829478.269617:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269618:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.269618:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269620:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829478.269622:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829478.269625:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829478.269627:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829478.269629:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829478.269632:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829478.269637:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327348096:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829478.269640:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269642:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.269644:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327348096/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269647:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829478.269650:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829478.269652:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829478.269674:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829478.269676:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829478.269677:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829478.269678:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829478.269680:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829478.269684:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829478.269685:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829478.269686:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829478.269688:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829478.269689:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269690:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269697:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829478.269699:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.269700:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829478.269701:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829478.269702:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829478.269705:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269713:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829478.269714:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.269715:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829478.269717:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829478.269718:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829478.269720:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348224:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829478.269721:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829478.269723:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829478.269724:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.269731:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829478.269733:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829478.269735:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829478.269742:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829478.269744:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829478.269747:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829478.269749:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348224, portal 10
00000100:00000001:1.0:1787829478.269751:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829478.269752:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829478.269757:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269761:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829478.269763:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829478.269764:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829478.269765:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327348224, offset 0
00000400:00000200:1.0:1787829478.269768:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829478.269770:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.269772:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829478.269774:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829478.269776:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829478.269777:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829478.269779:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829478.269781:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40492:40492:256:4294967295] 192.168.203.116@tcp LPNI seq info [40492:40492:8:4294967295]
00000400:00000200:1.0:1787829478.269784:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.269788:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829478.269791:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.269793:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829478.269796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829478.269798:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787829478.269800:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829478.269802:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787829478.269805:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.269806:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829478.269809:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829478.269813:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829478.269815:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829478.269817:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829478.269819:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829478.269822:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269830:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.269832:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269839:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.269841:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829478.269844:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829478.269847:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829478.269850:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829478.269852:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298298379
00000100:00000001:1.0:1787829478.269855:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298299052
00000100:00000001:1.0:1787829478.269857:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298298379
00000100:00000001:1.0:1787829478.269859:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298299052
00000100:00000001:1.0:1787829478.269861:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829478.269864:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.269866:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829478.269869:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829478/real 1787829478]  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829478.269877:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829478.269879:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829478.269880:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829478.269883:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829478.269885:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829478.269887:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829478.269889:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829478.269897:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269904:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829478.269905:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269907:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.269908:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829478.269912:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269919:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829478.269923:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269930:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829478.269932:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269934:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.269935:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.269937:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829478.269938:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829478.269940:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829478.269943:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829478.269944:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829478.269947:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829478.269952:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348224:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829478.269955:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829478.269958:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.269964:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829478.269966:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.269968:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829478.269969:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829478.269971:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829478.269974:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829478.269976:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829478.269977:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829478.269978:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.269980:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829478.269983:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829478.269984:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829478.269987:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829478.269989:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829478.269991:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829478.269993:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348352, portal 4
00000100:00000001:1.0:1787829478.269994:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829478.269995:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829478.270000:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270004:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829478.270006:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829478.270007:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829478.270009:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327348352, offset 0
00000400:00000200:1.0:1787829478.270011:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829478.270014:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.270016:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829478.270018:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829478.270019:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829478.270021:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829478.270022:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829478.270025:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40493:40493:256:4294967295] 192.168.203.116@tcp LPNI seq info [40493:40493:8:4294967295]
00000400:00000200:1.0:1787829478.270027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.270031:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829478.270034:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.270035:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829478.270038:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829478.270040:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787829478.270042:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829478.270044:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787829478.270046:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.270047:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829478.270049:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829478.270051:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829478.270053:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829478.270055:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829478.270056:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829478.270058:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270063:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.270065:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270082:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.270084:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829478.270087:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829478.270089:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829478.270091:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829478.270092:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298298386
00000100:00000001:1.0:1787829478.270094:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298299052
00000100:00000001:1.0:1787829478.270095:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298298386
00000100:00000001:1.0:1787829478.270097:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298299052
00000100:00000001:1.0:1787829478.270098:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829478.270099:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.270100:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829478.270103:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829478/real 1787829478]  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829478.270107:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829478.270108:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829478.270110:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829478.270111:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829478.270112:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829478.270113:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829478.270115:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829478.270121:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270126:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829478.270128:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270130:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.270131:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829478.270134:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270141:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829478.270144:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270152:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829478.270154:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270156:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.270157:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270159:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829478.270160:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829478.270163:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829478.270167:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829478.270168:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829478.270171:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829478.270176:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829478.270178:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829478.270180:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270183:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829478.270185:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.270186:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829478.270187:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829478.270189:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829478.270190:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348480:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829478.270192:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829478.270193:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829478.270194:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829478.270195:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829478.270198:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829478.270199:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829478.270201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829478.270203:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829478.270206:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829478.270208:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348480, portal 4
00000100:00000001:1.0:1787829478.270209:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829478.270210:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829478.270214:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270218:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829478.270219:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829478.270221:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829478.270222:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327348480, offset 0
00000400:00000200:1.0:1787829478.270224:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829478.270226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.270228:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829478.270230:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829478.270232:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829478.270233:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829478.270235:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829478.270237:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40494:40494:256:4294967295] 192.168.203.116@tcp LPNI seq info [40494:40494:8:4294967295]
00000400:00000200:1.0:1787829478.270239:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829478.270243:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829478.270246:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.270248:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829478.270250:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829478.270252:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787829478.270255:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829478.270256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787829478.270259:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829478.270260:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829478.270262:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829478.270264:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829478.270266:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829478.270267:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829478.270269:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829478.270271:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270276:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.270278:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270281:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.270283:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829478.270284:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829478.270286:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829478.270288:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829478.270289:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298298391
00000100:00000001:1.0:1787829478.270290:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298299052
00000100:00000001:1.0:1787829478.270292:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298298391
00000100:00000001:1.0:1787829478.270293:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298299052
00000100:00000001:1.0:1787829478.270295:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829478.270296:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.270297:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829478.270299:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829478/real 1787829478]  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829478.270302:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829478.270303:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829478.270304:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829478.270305:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829478.270306:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829478.270307:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829478.270308:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829478.270313:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270321:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829478.270322:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270323:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.270324:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829478.270326:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270329:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829478.270331:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270335:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829478.270336:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270337:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829478.270337:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270338:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829478.270339:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829478.270341:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829478.270342:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829478.270343:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829478.270345:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829478.270349:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348480:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829478.270351:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270352:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.270353:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327348224/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270358:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829478.270359:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829478.270360:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829478.270362:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829478.270363:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829478.270364:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829478.270365:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829478.270366:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829478.270368:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829478.270368:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829478.270369:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829478.270371:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829478.270372:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270373:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.270375:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327348352/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270379:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829478.270379:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829478.270380:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829478.270381:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829478.270382:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829478.270382:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829478.270383:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829478.270384:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829478.270385:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829478.270386:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829478.270386:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829478.270388:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829478.270388:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270389:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829478.270390:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327348480/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829478.270394:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829478.270394:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829478.270395:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829478.270396:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829478.270397:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829478.270397:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829478.270399:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829478.270400:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829478.270401:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829478.270402:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829478.270403:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829478.270404:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829478.270404:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270405:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829478.270407:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829478.270408:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829478.270409:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829478.270410:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829478.780731:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829478.780737:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829478.780739:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829478.780743:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829478.780746:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829478.780748:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829478.780750:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829478.780752:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829478.780754:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829478.972778:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298299052, at 4298299072
00000100:00000040:3.0:1787829478.972778:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298299052, at 4298299072
00000100:00000010:3.0:1787829478.972791:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63c60.
00000100:00000010:2.0:1787829478.972798:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e636c0.
00000020:00000001:2.0:1787829479.100895:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829479.100900:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829479.100905:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829479.804770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829479.804777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829479.804779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829479.804783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829479.804785:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829479.804788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829479.804791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829479.804793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829479.804796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829480.828731:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829480.828738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829480.828740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829480.828746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829480.828750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829480.828754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829480.828757:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829480.828760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829480.828763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829481.852775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829481.852782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829481.852784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829481.852790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829481.852794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829481.852797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829481.852800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829481.852803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829481.852805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829482.172846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829482.172849:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829482.172852:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829482.876743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829482.876750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829482.876752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829482.876758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829482.876762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829482.876766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829482.876770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829482.876772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829482.876776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829483.196861:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829483.196867:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829483.196873:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829483.388955:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829483.388960:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829483.388965:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829483.388974:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.388992:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829483.388995:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.388999:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829483.389003:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829483.389005:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829483.389013:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327348608:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829483.389018:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829483.389022:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829483.389025:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.389030:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829483.389071:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829483.389082:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829483.389093:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829483.389100:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829483.389106:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829483.389115:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348608, portal 25
00000100:00000001:1.0:1787829483.389120:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829483.389123:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829483.389133:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389141:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829483.389146:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829483.389148:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829483.389153:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327348608, offset 0
00000400:00000200:1.0:1787829483.389163:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829483.389173:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.389181:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829483.389189:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829483.389197:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829483.389202:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829483.389206:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829483.389214:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40495:40495:256:4294967295] 192.168.203.116@tcp LPNI seq info [40495:40495:8:4294967295]
00000400:00000200:1.0:1787829483.389221:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.389229:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829483.389237:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.389242:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829483.389252:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829483.389256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787829483.389265:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829483.389270:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787829483.389279:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.389283:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829483.389289:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829483.389296:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829483.389301:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829483.389304:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829483.389310:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829483.389316:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389332:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.389337:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389348:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.389350:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829483.389357:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829483.389363:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829483.389368:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298304172
00000100:00000010:1.0:1787829483.389377:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e634e0.
00000100:00000001:1.0:1787829483.389394:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829483.389404:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298304172
00000100:00000010:1.0:1787829483.389412:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63720.
00000100:00000001:1.0:1787829483.389419:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829483.389431:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829483.389435:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.389440:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829483.389449:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483]  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829483.389462:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829483.389465:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829483.389469:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829483.389474:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829483.389478:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829483.389483:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829483.389491:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829483.389503:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389513:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829483.389516:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.389519:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.389522:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829483.389529:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389544:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829483.389552:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389565:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829483.389569:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.389572:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.389576:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.389580:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829483.389583:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829483.389586:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829483.389594:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829483.389598:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829483.389610:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829483.389630:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327348608:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829483.389639:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.389644:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.389652:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327348608/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389702:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829483.389712:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829483.389716:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829483.389722:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829483.389725:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829483.389728:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829483.389733:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829483.389736:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829483.389741:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829483.389744:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829483.389747:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829483.389750:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829483.389754:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.389755:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.389761:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829483.389763:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.389769:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829483.389771:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829483.389773:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829483.389778:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389800:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829483.389803:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.389805:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829483.389808:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829483.389811:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829483.389817:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348736:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829483.389820:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829483.389823:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829483.389827:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.389831:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829483.389837:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829483.389842:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829483.389850:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829483.389854:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829483.389860:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829483.389864:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348736, portal 10
00000100:00000001:1.0:1787829483.389867:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829483.389869:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829483.389877:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389882:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829483.389884:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829483.389887:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829483.389889:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327348736, offset 0
00000400:00000200:1.0:1787829483.389891:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829483.389895:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.389898:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829483.389900:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829483.389902:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829483.389904:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829483.389906:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829483.389908:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40496:40496:256:4294967295] 192.168.203.116@tcp LPNI seq info [40496:40496:8:4294967295]
00000400:00000200:1.0:1787829483.389910:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.389915:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829483.389918:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.389920:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829483.389923:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829483.389925:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787829483.389928:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829483.389930:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787829483.389933:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.389934:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829483.389937:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829483.389939:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829483.389941:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829483.389942:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829483.389944:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829483.389947:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389952:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.389954:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.389959:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.389960:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829483.389962:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829483.389966:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829483.389968:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829483.389969:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298303499
00000100:00000001:1.0:1787829483.389971:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298304172
00000100:00000001:1.0:1787829483.389972:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298303499
00000100:00000001:1.0:1787829483.389974:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298304172
00000100:00000001:1.0:1787829483.389975:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829483.389977:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.389978:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829483.389980:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483]  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829483.389985:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829483.389986:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829483.389988:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829483.389989:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829483.389991:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829483.389992:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829483.389994:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829483.389999:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390003:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829483.390004:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390005:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.390006:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829483.390008:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390012:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829483.390015:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390020:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829483.390022:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390023:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.390024:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390025:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829483.390027:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829483.390028:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829483.390030:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829483.390031:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829483.390051:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829483.390057:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348736:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829483.390059:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829483.390061:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390065:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829483.390066:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.390068:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829483.390069:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829483.390071:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829483.390073:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829483.390075:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829483.390076:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829483.390077:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.390078:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829483.390081:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829483.390082:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829483.390085:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829483.390087:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829483.390089:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829483.390092:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348864, portal 4
00000100:00000001:1.0:1787829483.390093:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829483.390094:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829483.390098:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390102:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829483.390104:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829483.390105:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829483.390107:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327348864, offset 0
00000400:00000200:1.0:1787829483.390109:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829483.390112:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.390114:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829483.390116:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829483.390118:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829483.390120:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829483.390121:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829483.390123:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40497:40497:256:4294967295] 192.168.203.116@tcp LPNI seq info [40497:40497:8:4294967295]
00000400:00000200:1.0:1787829483.390126:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.390130:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829483.390133:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.390135:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829483.390138:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829483.390140:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787829483.390143:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829483.390144:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787829483.390147:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.390148:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829483.390150:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829483.390153:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829483.390154:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829483.390156:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829483.390158:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829483.390160:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390165:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.390167:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390171:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.390172:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829483.390174:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829483.390176:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829483.390178:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829483.390179:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298303506
00000100:00000001:1.0:1787829483.390181:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298304172
00000100:00000001:1.0:1787829483.390183:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298303506
00000100:00000001:1.0:1787829483.390184:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298304172
00000100:00000001:1.0:1787829483.390185:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829483.390186:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.390187:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829483.390190:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483]  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829483.390194:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829483.390195:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829483.390196:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829483.390198:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829483.390199:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829483.390200:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829483.390202:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829483.390208:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390216:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829483.390217:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390218:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.390219:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829483.390221:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390225:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829483.390227:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390231:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829483.390233:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390234:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.390235:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390236:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829483.390237:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829483.390239:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829483.390241:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829483.390242:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829483.390245:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829483.390249:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829483.390252:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390253:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.390255:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327348736/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390258:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829483.390259:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829483.390261:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829483.390263:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829483.390264:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829483.390265:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829483.390266:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829483.390268:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:1.0:1787829483.390269:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829483.390270:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829483.390271:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829483.390273:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829483.390274:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390275:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.390277:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327348864/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390281:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829483.390281:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829483.390282:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829483.390284:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829483.390284:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829483.390285:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829483.390286:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829483.390287:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829483.390288:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829483.390289:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829483.390289:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829483.390291:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829483.390292:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390293:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390295:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829483.390296:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.390297:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829483.390298:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829483.390298:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829483.390300:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390304:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829483.390306:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.390307:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829483.390308:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829483.390310:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829483.390312:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829483.390313:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829483.390315:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829483.390316:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829483.390317:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829483.390318:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829483.390319:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829483.390321:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829483.390323:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829483.390325:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829483.390326:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327348992, portal 4
00000100:00000001:1.0:1787829483.390328:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829483.390329:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829483.390332:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390336:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829483.390338:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829483.390339:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829483.390341:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327348992, offset 0
00000400:00000200:1.0:1787829483.390343:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829483.390345:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.390347:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829483.390349:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829483.390350:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829483.390352:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829483.390353:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829483.390355:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40498:40498:256:4294967295] 192.168.203.116@tcp LPNI seq info [40498:40498:8:4294967295]
00000400:00000200:1.0:1787829483.390358:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829483.390362:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829483.390365:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.390367:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829483.390370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829483.390371:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787829483.390374:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829483.390376:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787829483.390379:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829483.390380:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:1.0:1787829483.390382:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829483.390384:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829483.390386:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:1.0:1787829483.390387:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:1.0:1787829483.390388:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829483.390391:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390396:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.390397:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390402:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.390403:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829483.390404:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829483.390406:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829483.390408:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829483.390410:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298303511
00000100:00000001:1.0:1787829483.390411:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298304172
00000100:00000001:1.0:1787829483.390413:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298303511
00000100:00000001:1.0:1787829483.390414:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298304172
00000100:00000001:1.0:1787829483.390415:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829483.390417:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.390418:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829483.390420:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483]  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829483.390425:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829483.390426:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829483.390427:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829483.390429:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829483.390430:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829483.390431:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829483.390433:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829483.390438:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390441:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829483.390442:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390444:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.390445:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829483.390447:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390451:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829483.390453:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390457:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829483.390458:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390459:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829483.390460:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390461:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829483.390462:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829483.390464:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829483.390466:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829483.390467:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829483.390470:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829483.390475:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327348992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829483.390477:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390478:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829483.390480:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327348992/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829483.390484:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829483.390485:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829483.390486:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829483.390487:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829483.390488:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829483.390488:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829483.390489:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829483.390490:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829483.390492:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829483.390492:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829483.390493:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829483.390494:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829483.390495:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390496:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829483.390498:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829483.390499:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829483.390500:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829483.390501:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829483.900782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829483.900795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829483.900798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829483.900805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829483.900809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829483.900813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829483.900816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829483.900818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829483.900822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829484.092756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298304172, at 4298304192
00000100:00000040:3.0:1787829484.092762:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298304172, at 4298304192
00000100:00000010:3.0:1787829484.092779:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63720.
00000100:00000010:2.0:1787829484.092784:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e634e0.
00000100:00000001:2.0:1787829484.924788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829484.924795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829484.924798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829484.924802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829484.924805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829484.924809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829484.924812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829484.924815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829484.924818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829485.244851:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829485.244856:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829485.244860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829485.948805:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829485.948815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829485.948818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829485.948825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829485.948829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829485.948834:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829485.948837:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829485.948841:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829485.948844:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829486.268797:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829486.268800:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829486.268808:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829486.972823:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829486.972834:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829486.972837:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829486.972844:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829486.972848:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829486.972852:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829486.972855:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829486.972858:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829486.972860:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829487.996810:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829487.996817:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829487.996819:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829487.996825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829487.996829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829487.996835:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829487.996838:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829487.996841:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829487.996844:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829488.316885:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829488.316890:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829488.316894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829488.508990:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829488.509007:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829488.509012:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829488.509020:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509032:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829488.509035:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509040:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829488.509047:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829488.509053:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829488.509062:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327349120:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829488.509069:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829488.509076:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829488.509080:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509087:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829488.509096:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829488.509102:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829488.509111:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829488.509120:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829488.509131:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829488.509143:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327349120, portal 25
00000100:00000001:1.0:1787829488.509149:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829488.509154:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829488.509167:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509189:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829488.509196:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829488.509211:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98cc0.
00000100:00000200:1.0:1787829488.509220:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327349120, offset 0
00000400:00000200:1.0:1787829488.509235:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509249:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509260:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829488.509272:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829488.509282:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829488.509291:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829488.509299:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509308:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40499:40499:256:4294967295] 192.168.203.116@tcp LPNI seq info [40499:40499:8:4294967295]
00000400:00000200:1.0:1787829488.509314:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509332:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829488.509343:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829488.509352:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955b00.
00000800:00000200:1.0:1787829488.509363:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829488.509372:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829488.509387:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829488.509395:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829488.509398:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298309292
00000100:00000010:1.0:1787829488.509404:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63360.
00000100:00000001:1.0:1787829488.509412:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829488.509419:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298309292
00000100:00000010:1.0:1787829488.509423:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e638a0.
00000100:00000001:1.0:1787829488.509430:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829488.509440:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829488.509445:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509451:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509454:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829488.509464:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829488.509467:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829488.509470:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829488.509472:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829488.509474:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829488.509480:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509490:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829488.509492:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509493:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829488.509495:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829488.509496:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829488.509498:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327349248:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829488.509502:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829488.509504:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829488.509505:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509506:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829488.509509:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829488.509511:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829488.509514:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829488.509516:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630d68.
00000400:00000010:1.0:1787829488.509518:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829488.509521:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327349248, portal 10
00000100:00000001:1.0:1787829488.509523:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829488.509524:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829488.509530:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509537:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829488.509539:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829488.509542:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829488.509545:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327349248, offset 0
00000400:00000200:1.0:1787829488.509550:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509554:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509559:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829488.509561:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829488.509564:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829488.509567:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829488.509570:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509573:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40500:40500:255:4294967295] 192.168.203.116@tcp LPNI seq info [40500:40500:7:4294967295]
00000400:00000200:1.0:1787829488.509576:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509582:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829488.509586:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829488.509590:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955d00.
00000800:00000200:1.0:1787829488.509594:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829488.509598:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829488.509600:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829488.509603:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829488.509606:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829488.509608:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298308618
00000100:00000001:1.0:1787829488.509611:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298309292
00000100:00000001:1.0:1787829488.509613:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298308618
00000100:00000001:1.0:1787829488.509615:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298309292
00000100:00000001:1.0:1787829488.509617:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829488.509619:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509621:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829488.509624:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509631:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829488.509632:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509634:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829488.509636:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829488.509637:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829488.509640:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327349376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829488.509643:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829488.509645:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829488.509646:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509648:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829488.509651:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829488.509653:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829488.509681:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829488.509685:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829488.509689:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829488.509691:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327349376, portal 4
00000100:00000001:1.0:1787829488.509694:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829488.509695:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829488.509701:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509708:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829488.509711:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829488.509713:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f985d8.
00000100:00000200:1.0:1787829488.509715:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327349376, offset 0
00000400:00000200:1.0:1787829488.509719:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509723:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509727:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829488.509730:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829488.509732:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829488.509734:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829488.509736:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509739:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40501:40501:254:4294967295] 192.168.203.116@tcp LPNI seq info [40501:40501:6:4294967295]
00000400:00000200:1.0:1787829488.509743:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509749:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829488.509752:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829488.509756:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829488.509759:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829488.509763:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829488.509765:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829488.509768:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829488.509770:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829488.509772:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298308626
00000100:00000001:1.0:1787829488.509774:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298309292
00000100:00000001:1.0:1787829488.509777:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298308626
00000100:00000001:1.0:1787829488.509779:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298309292
00000100:00000001:1.0:1787829488.509781:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829488.509783:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509785:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509787:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829488.509790:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829488.509792:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829488.509794:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829488.509795:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829488.509796:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829488.509800:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509807:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829488.509809:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509811:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829488.509812:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829488.509814:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829488.509816:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327349504:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829488.509820:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829488.509821:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829488.509823:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829488.509825:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829488.509828:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829488.509830:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829488.509832:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829488.509834:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd946304d0.
00000400:00000010:1.0:1787829488.509837:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98e58.
00000100:00000200:1.0:1787829488.509840:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327349504, portal 4
00000100:00000001:1.0:1787829488.509842:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829488.509843:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829488.509848:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829488.509854:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829488.509856:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829488.509858:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98aa0.
00000100:00000200:1.0:1787829488.509860:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327349504, offset 0
00000400:00000200:1.0:1787829488.509864:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509867:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509870:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829488.509873:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829488.509875:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829488.509877:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829488.509879:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829488.509882:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40502:40502:253:4294967295] 192.168.203.116@tcp LPNI seq info [40502:40502:5:4294967295]
00000400:00000200:1.0:1787829488.509885:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829488.509891:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829488.509895:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829488.509898:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955900.
00000800:00000200:1.0:1787829488.509902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829488.509905:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829488.509908:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829488.509911:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829488.509915:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829488.509916:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298308631
00000100:00000001:1.0:1787829488.509918:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298309292
00000100:00000001:1.0:1787829488.509920:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298308631
00000100:00000001:1.0:1787829488.509922:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298309292
00000100:00000001:1.0:1787829488.509924:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829488.509926:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509928:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509930:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829488.509933:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829488.509934:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829488.509935:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829488.509937:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829488.509938:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829488.509940:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829489.020788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829489.020798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829489.020801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829489.020808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829489.020813:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829489.020817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829489.020820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829489.020824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829489.020827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829489.212704:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298309292, at 4298309312
00000100:00000010:3.0:1787829489.212720:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e638a0.
00000100:00000040:2.0:1787829489.212725:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298309292, at 4298309312
00000100:00000010:2.0:1787829489.212743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63360.
00000020:00000001:3.0:1787829489.340810:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829489.340814:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829489.340818:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829490.044804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829490.044815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829490.044816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829490.044823:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829490.044827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829490.044832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829490.044835:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829490.044838:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829490.044841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829491.068823:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829491.068832:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829491.068836:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829491.068842:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829491.068847:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829491.068852:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829491.068855:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829491.068858:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829491.068861:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829491.388868:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829491.388873:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829491.388876:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829491.581029:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829491.581087:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829491.581099:0:2337:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829491.581103:0:2337:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829491.581108:0:2337:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829491.581112:0:2337:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829491.581118:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829491.581122:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955b00.
00000400:00000200:2.0:1787829491.581129:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829491.581134:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829491.581143:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98cc0
00000400:00000010:2.0:1787829491.581146:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98cc0.
00000100:00000001:2.0:1787829491.581153:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829491.581172:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581200:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829491.581205:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581213:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829491.581215:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829491.581227:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829491.581229:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955d00.
00000400:00000200:2.0:1787829491.581232:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829491.581236:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829491.581239:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000400:00000010:2.0:1787829491.581241:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00000001:2.0:1787829491.581243:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829491.581248:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581257:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829491.581260:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581266:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829491.581268:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829491.581273:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829491.581276:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:2.0:1787829491.581279:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829491.581282:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829491.581284:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f985d8
00000400:00000010:2.0:1787829491.581286:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f985d8.
00000100:00000001:2.0:1787829491.581288:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829491.581291:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581297:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829491.581299:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581303:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829491.581305:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829491.581309:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829491.581311:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955900.
00000400:00000200:2.0:1787829491.581314:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829491.581317:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829491.581320:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98aa0
00000400:00000010:2.0:1787829491.581321:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98aa0.
00000100:00000001:2.0:1787829491.581324:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829491.581329:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581336:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829491.581339:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829491.581345:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829491.581346:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829491.581718:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829491.581727:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829491.581731:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829491.581748:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829491]  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829491.581768:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829491.581770:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829491.581774:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd946304d0.
00000400:00000200:1.0:1787829491.581780:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98e58
00000400:00000010:1.0:1787829491.581783:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98e58.
00000100:00000001:1.0:1787829491.581788:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829491.581793:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829491.581805:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.581814:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829491.581816:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.581819:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.581821:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829491.581826:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.581833:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829491.581839:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.581846:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829491.581849:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.581850:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.581852:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.581854:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829491.581857:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829491.581860:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829491.581865:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829491.581870:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829491.581876:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829491.581885:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327349504:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829491.581892:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829491.581902:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829491]  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829491.581918:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829491.581919:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829491.581922:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630d68.
00000400:00000200:1.0:1787829491.581925:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829491.581927:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829491.581930:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829491.581933:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829491.581942:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.581949:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829491.581951:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.581953:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.581954:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829491.581958:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.581965:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829491.581970:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.581991:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829491.581995:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.581997:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.581998:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582000:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829491.582003:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829491.582007:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829491.582011:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829491.582015:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829491.582021:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829491.582031:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327349248:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829491.582036:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829491.582040:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829491]  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829491.582048:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829491.582050:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829491.582052:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829491.582055:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829491.582057:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829491.582060:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829491.582063:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829491.582071:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582077:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829491.582079:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582081:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.582083:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829491.582089:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582096:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829491.582101:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582107:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829491.582109:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582111:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.582113:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582114:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829491.582116:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829491.582118:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829491.582122:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829491.582124:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829491.582127:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829491.582135:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327349376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829491.582138:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829491.582142:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829491]  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829491.582150:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829491.582151:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829491.582154:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829491.582156:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829491.582158:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829491.582160:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829491.582164:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829491.582173:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582180:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829491.582183:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582185:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.582186:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829491.582192:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582206:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829491.582210:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582230:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829491.582232:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582234:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829491.582236:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582237:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829491.582239:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829491.582242:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829491.582246:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829491.582248:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829491.582254:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829491.582262:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327349120:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829491.582267:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582269:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829491.582272:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327349504/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582277:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829491.582279:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829491.582282:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829491.582286:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829491.582289:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829491.582291:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829491.582294:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829491.582296:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829491.582300:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829491.582303:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829491.582305:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829491.582309:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829491.582310:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582312:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829491.582316:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327349248/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582323:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829491.582324:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829491.582326:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829491.582329:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829491.582330:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829491.582331:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829491.582334:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829491.582336:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829491.582338:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829491.582339:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829491.582341:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829491.582344:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829491.582345:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582347:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829491.582351:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327349376/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582357:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829491.582358:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829491.582360:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829491.582363:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829491.582364:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829491.582366:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829491.582368:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829491.582369:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:1.0:1787829491.582372:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829491.582373:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829491.582374:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829491.582377:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829491.582378:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582381:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829491.582384:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327349120/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829491.582391:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829491.582393:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829491.582394:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829491.582397:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829491.582398:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829491.582399:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829491.582402:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829491.582403:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829491.582405:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829491.582406:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829491.582408:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829491.582410:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829491.582411:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582413:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829491.582420:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829491.582423:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829491.582425:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829491.582427:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829492.092769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829492.092778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829492.092781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829492.092788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829492.092793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829492.092797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829492.092799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829492.092801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829492.092804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829492.412852:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829492.412856:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829492.412858:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829493.116802:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829493.116813:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829493.116816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829493.116831:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829493.116836:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829493.116841:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829493.116845:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829493.116847:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829493.116851:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829493.628793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829493.628804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829493.628806:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829493.628810:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829493.628814:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829493.628826:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3641
00000400:00000200:1.0:1787829493.628835:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829493.628841:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829493.628844:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829493.628846:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.628849:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.628851:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.628853:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829493.628856:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829493.628858:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829493.628860:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829493.628864:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829493.628865:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829493.628867:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829493.628868:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.628870:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.628871:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.628872:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829493.628874:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829493.628877:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829493.628878:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829493.628881:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829493.628883:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829493.628885:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.628887:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829493.628888:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829493.628895:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829493.628898:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829493.628901:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829493.628903:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.628906:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829493.628909:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829493.628913:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829493.628920:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829493.628928:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829493.628940:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327349632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829493.628952:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829493.628971:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829493.628977:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327349632/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829493.629005:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829493.629009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829493.629012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829493.629014:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829493.629015:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829493.629018:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829493.629021:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3641
00000400:00000200:1.0:1787829493.629025:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829493.629029:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829493.629031:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829493.629035:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629037:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.629040:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629053:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829493.629057:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829493.629060:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829493.629062:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829493.629066:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829493.629071:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829493.629075:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829493.629081:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629085:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.629088:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629091:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829493.629095:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829493.629105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629111:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829493.629118:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829493.629127:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829493.629133:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629136:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829493.629148:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829493.629154:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629160:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829493.629170:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829493.629176:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829493.629180:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829493.629181:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629184:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829493.629187:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829493.629190:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829493.629196:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829493.629203:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829493.629210:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327349760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829493.629223:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829493.629228:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829493.629236:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327349760/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829493.629251:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829493.629255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829493.629261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829493.629265:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829493.629267:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829493.629271:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829493.629275:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3641
00000400:00000200:1.0:1787829493.629325:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829493.629331:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829493.629335:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829493.629342:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629346:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.629350:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629353:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829493.629357:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829493.629358:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829493.629360:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829493.629362:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829493.629366:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829493.629368:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829493.629373:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629377:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.629380:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629383:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829493.629389:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787829493.629393:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829493.629394:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629401:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829493.629401:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
02000000:00000001:1.0:1787829493.629405:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829493.629409:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829493.629414:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629418:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829493.629422:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829493.629426:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629430:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829493.629436:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829493.629440:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829493.629443:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829493.629447:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629452:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829493.629455:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829493.629459:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829493.629480:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829493.629488:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829493.629496:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327349888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829493.629506:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829493.629511:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829493.629519:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327349888/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829493.629534:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829493.629538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829493.629546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829493.629549:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829493.629551:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829493.629556:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829493.629561:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3641
00000400:00000200:1.0:1787829493.629569:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829493.629576:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829493.629581:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829493.629587:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629591:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.629596:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629600:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829493.629605:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829493.629608:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829493.629611:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829493.629615:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829493.629630:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829493.629635:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829493.629642:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629648:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829493.629653:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629695:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829493.629701:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829493.629708:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829493.629713:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829493.629717:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829493.629721:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829493.629725:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629730:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829493.629734:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829493.629739:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629742:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829493.629748:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829493.629753:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829493.629759:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829493.629762:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829493.629765:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829493.629769:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829493.629773:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829493.629780:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829493.629786:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829493.629795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327350016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829493.629810:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829493.629815:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829493.629824:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327350016/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829493.629837:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829493.629846:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3651)
00000800:00000200:0.1:1787829493.630029:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829493.630032:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829493.630291:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829493.630293:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829493.630527:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829493.630528:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787829494.140744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829494.140752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829494.140754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829494.140759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829494.140762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829494.140765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829494.140767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829494.140769:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829494.140771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829494.332724:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298314412, at 4298314432
00000100:00000040:3.0:1787829494.332724:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298314412, at 4298314432
00000100:00000010:2.0:1787829494.332743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000100:00000010:3.0:1787829494.332750:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000020:00000001:2.0:1787829494.460875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829494.460879:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829494.460881:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829495.164793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829495.164804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829495.164806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829495.164814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829495.164818:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829495.164823:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829495.164827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829495.164830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829495.164833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829495.484795:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829495.484798:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829495.484801:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829496.188743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829496.188751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829496.188753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829496.188760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829496.188763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829496.188766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829496.188769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829496.188771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829496.188773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829497.212763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829497.212772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829497.212775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829497.212781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829497.212785:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829497.212790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829497.212792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829497.212795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829497.212799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829497.532824:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829497.532828:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829497.532830:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829498.236714:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829498.236725:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829498.236727:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829498.236736:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829498.236741:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829498.236745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829498.236748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829498.236751:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829498.236755:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829498.556887:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829498.556891:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829498.556895:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829498.748765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829498.748775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829498.748779:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829498.748783:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829498.748787:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829498.748795:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3646
00000400:00000200:1.0:1787829498.748806:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829498.748813:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829498.748817:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829498.748821:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748825:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.748827:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748830:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829498.748835:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829498.748837:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829498.748840:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829498.748844:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829498.748845:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829498.748847:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829498.748850:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748851:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.748852:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748854:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829498.748856:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829498.748859:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748860:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829498.748862:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829498.748865:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829498.748867:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.748869:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829498.748872:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829498.748881:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829498.748884:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829498.748888:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829498.748890:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.748892:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829498.748895:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829498.748900:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829498.748907:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829498.748913:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829498.748922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327350144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829498.748941:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829498.748945:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829498.748949:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327350144/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829498.748968:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829498.748970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829498.748972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829498.748974:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829498.748975:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829498.748980:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829498.748981:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3646
00000400:00000200:1.0:1787829498.748984:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829498.748986:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829498.748988:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829498.748990:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748992:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.748993:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.748995:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829498.748997:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829498.748998:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829498.748999:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829498.749000:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829498.749002:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829498.749004:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829498.749006:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749008:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.749009:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749011:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829498.749012:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829498.749017:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749019:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829498.749023:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829498.749028:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829498.749033:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749035:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829498.749041:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829498.749045:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749049:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829498.749053:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829498.749056:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829498.749060:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829498.749063:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749067:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829498.749069:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829498.749071:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829498.749076:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829498.749080:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829498.749087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327350272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829498.749096:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829498.749098:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829498.749103:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327350272/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829498.749124:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829498.749130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829498.749137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829498.749140:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829498.749144:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829498.749150:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829498.749157:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3646
00000400:00000200:1.0:1787829498.749163:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829498.749166:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829498.749168:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829498.749170:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749174:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.749179:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749182:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829498.749185:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829498.749188:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829498.749192:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829498.749197:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829498.749202:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829498.749208:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829498.749216:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749223:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.749229:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749235:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829498.749238:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829498.749247:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749253:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829498.749258:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829498.749263:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829498.749269:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749273:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829498.749277:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.1:1787829498.749280:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:1.0:1787829498.749281:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749284:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829498.749286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000010:1.0:1787829498.749289:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829498.749294:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829498.749298:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829498.749301:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749302:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829498.749304:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829498.749307:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829498.749312:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829498.749318:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829498.749325:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327350400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829498.749345:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829498.749348:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829498.749355:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327350400/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829498.749383:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829498.749389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829498.749395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829498.749398:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829498.749401:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829498.749408:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829498.749415:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3646
00000400:00000200:1.0:1787829498.749422:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829498.749427:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829498.749430:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829498.749434:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749437:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.749441:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749443:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829498.749446:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829498.749447:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829498.749450:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829498.749452:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829498.749456:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829498.749459:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829498.749465:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749470:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829498.749476:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749483:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829498.749489:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829498.749498:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829498.749504:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829498.749510:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829498.749515:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829498.749522:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749528:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829498.749533:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829498.749540:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749545:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829498.749553:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829498.749559:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829498.749566:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829498.749570:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829498.749575:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829498.749580:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829498.749582:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829498.749589:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829498.749597:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829498.749607:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327350528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829498.749627:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829498.749632:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829498.749643:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327350528/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829498.749686:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829498.749695:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3656)
00000800:00000200:0.1:1787829498.749798:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829498.749799:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829498.750047:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829498.750048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829498.750216:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829498.750218:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829499.260732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829499.260741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829499.260744:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829499.260749:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829499.260753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829499.260757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829499.260760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829499.260763:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829499.260766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829499.452752:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298319532, at 4298319552
00000100:00000010:3.0:1787829499.452768:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000040:2.0:1787829499.452832:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298319532, at 4298319552
00000100:00000010:2.0:1787829499.452837:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000001:2.0:1787829500.284724:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829500.284733:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829500.284735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829500.284739:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829500.284743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829500.284746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829500.284748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829500.284750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829500.284753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829500.604825:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829500.604829:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829500.604832:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829501.308808:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829501.308817:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829501.308818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829501.308824:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829501.308827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829501.308830:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829501.308833:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829501.308835:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829501.308838:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829501.628832:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829501.628836:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829501.628839:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829502.332768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829502.332778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829502.332782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829502.332789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829502.332793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829502.332798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829502.332801:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829502.332804:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829502.332807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829503.356782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829503.356791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829503.356794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829503.356801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829503.356805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829503.356810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829503.356812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829503.356816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829503.356820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829503.676865:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829503.676868:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829503.676870:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829503.868853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829503.868863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829503.868865:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829503.868868:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829503.868873:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829503.868881:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3651
00000400:00000200:3.0:1787829503.868903:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829503.868911:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829503.868914:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829503.868916:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.868919:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.868920:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.868922:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829503.868925:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829503.868927:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829503.868930:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829503.868933:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829503.868935:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829503.868936:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829503.868938:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.868940:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.868942:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.868943:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829503.868946:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829503.868948:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829503.868950:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829503.868952:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829503.868955:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829503.868957:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.868959:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829503.868961:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829503.868972:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829503.868974:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829503.868978:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829503.868979:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.868982:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829503.868985:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829503.868990:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829503.868998:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829503.869004:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829503.869013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327350656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829503.869021:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829503.869026:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829503.869029:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327350656/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829503.869048:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829503.869050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829503.869052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829503.869053:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829503.869055:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829503.869057:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829503.869059:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3651
00000400:00000200:3.0:1787829503.869061:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829503.869063:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829503.869065:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829503.869067:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869068:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.869070:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869072:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829503.869073:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829503.869074:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829503.869075:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829503.869076:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829503.869077:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829503.869079:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829503.869083:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869086:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.869089:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869092:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829503.869094:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829503.869098:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869100:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829503.869103:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829503.869110:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829503.869114:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869116:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829503.869121:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829503.869124:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869127:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829503.869133:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829503.869135:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829503.869139:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829503.869142:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869145:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829503.869146:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829503.869150:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829503.869153:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829503.869157:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829503.869163:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327350784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829503.869171:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829503.869174:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829503.869178:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327350784/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829503.869185:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829503.869187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829503.869190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829503.869191:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829503.869192:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829503.869194:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829503.869197:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3651
00000400:00000200:3.0:1787829503.869201:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829503.869205:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829503.869207:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829503.869211:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869215:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.869218:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869222:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829503.869225:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829503.869229:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829503.869232:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829503.869234:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829503.869237:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829503.869241:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829503.869244:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869247:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.869248:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869249:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829503.869251:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829503.869254:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869259:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829503.869259:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829503.869261:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829503.869262:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869263:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829503.869265:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829503.869266:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869270:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829503.869275:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829503.869277:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829503.869279:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829503.869280:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869282:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829503.869285:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829503.869286:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829503.869290:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829503.869294:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829503.869299:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327350912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829503.869308:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829503.869310:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829503.869314:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327350912/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829503.869330:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829503.869335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829503.869341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829503.869345:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829503.869346:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829503.869349:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829503.869354:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3651
00000400:00000200:3.0:1787829503.869370:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829503.869376:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829503.869379:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829503.869381:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869387:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.869388:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869389:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829503.869390:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829503.869391:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829503.869393:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829503.869397:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829503.869400:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829503.869403:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829503.869405:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869408:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829503.869412:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869414:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829503.869416:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000200:0.1:1787829503.869418:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829503.869419:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829503.869424:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829503.869425:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
02000000:00000001:3.0:1787829503.869426:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829503.869428:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829503.869431:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869433:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829503.869436:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829503.869439:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869441:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829503.869446:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829503.869449:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829503.869450:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829503.869452:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829503.869456:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829503.869458:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829503.869459:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829503.869464:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829503.869468:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829503.869474:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327351040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829503.869482:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829503.869485:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829503.869489:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327351040/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829503.869495:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829503.869499:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3661)
00000800:00000200:0.1:1787829503.869888:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829503.869905:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829503.870087:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829503.870088:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829503.870245:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829503.870247:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829504.380798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829504.380806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829504.380808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829504.380813:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829504.380817:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829504.380820:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829504.380822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829504.380824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829504.380827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829504.572708:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298324652, at 4298324672
00000100:00000040:2.0:1787829504.572714:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298324652, at 4298324672
00000100:00000010:3.0:1787829504.572723:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000100:00000010:2.0:1787829504.572726:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000020:00000001:3.0:1787829504.700843:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829504.700846:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829504.700849:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829505.404776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829505.404783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829505.404785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829505.404790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829505.404794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829505.404797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829505.404800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829505.404802:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829505.404804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829506.428784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829506.428792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829506.428794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829506.428799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829506.428802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829506.428805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829506.428808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829506.428811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829506.428813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829506.748839:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829506.748842:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829506.748845:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829507.452753:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829507.452759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829507.452760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829507.452764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829507.452767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829507.452769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829507.452771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829507.452774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829507.452776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829507.772803:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829507.772806:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829507.772809:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829508.476767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829508.476772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829508.476774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829508.476778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829508.476781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829508.476784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829508.476786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829508.476788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829508.476790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829508.988765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829508.988775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829508.988779:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829508.988783:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829508.988788:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829508.988795:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3656
00000400:00000200:1.0:1787829508.988804:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829508.988810:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829508.988813:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829508.988815:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988818:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.988820:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988822:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829508.988825:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829508.988826:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829508.988829:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829508.988832:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829508.988833:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829508.988835:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829508.988837:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988838:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.988840:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988841:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829508.988843:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829508.988846:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988848:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829508.988850:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829508.988866:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829508.988869:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.988871:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829508.988873:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829508.988882:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829508.988885:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829508.988889:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829508.988890:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.988893:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829508.988896:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829508.988900:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829508.988909:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829508.988916:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829508.988925:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327351168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829508.988934:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829508.988938:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829508.988941:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327351168/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829508.988959:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829508.988961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829508.988963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829508.988965:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829508.988966:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829508.988968:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829508.988970:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3656
00000400:00000200:1.0:1787829508.988973:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829508.988975:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829508.988976:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829508.988978:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988980:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.988981:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988983:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829508.988985:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829508.988986:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829508.988987:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829508.988988:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829508.988989:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829508.988991:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829508.988992:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988994:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.988995:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.988997:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829508.988999:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829508.989000:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989002:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829508.989004:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829508.989015:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829508.989018:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989020:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829508.989025:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829508.989029:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989032:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829508.989037:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829508.989040:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829508.989043:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829508.989046:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989048:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829508.989049:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829508.989051:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829508.989055:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829508.989060:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829508.989065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327351296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829508.989074:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829508.989077:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829508.989083:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327351296/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829508.989094:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829508.989096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829508.989098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829508.989099:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829508.989102:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829508.989105:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829508.989109:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3656
00000400:00000200:1.0:1787829508.989113:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829508.989115:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829508.989117:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829508.989120:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989126:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.989129:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989131:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829508.989133:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829508.989133:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829508.989134:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829508.989136:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829508.989138:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829508.989142:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829508.989144:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989146:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.989149:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989151:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829508.989153:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829508.989154:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989155:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829508.989157:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829508.989159:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829508.989166:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989167:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829508.989169:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829508.989173:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989175:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829508.989179:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829508.989182:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829508.989185:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829508.989186:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989187:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829508.989188:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829508.989190:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829508.989193:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829508.989197:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829508.989202:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327351424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829508.989207:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829508.989210:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829508.989215:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327351424/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829508.989223:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829508.989226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829508.989229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829508.989231:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829508.989233:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829508.989236:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829508.989239:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3656
00000400:00000200:1.0:1787829508.989243:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829508.989245:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829508.989247:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829508.989250:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989253:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.989258:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989262:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829508.989264:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829508.989266:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829508.989269:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829508.989273:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829508.989276:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829508.989281:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829508.989288:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989292:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829508.989297:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989299:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829508.989302:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829508.989306:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829508.989310:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829508.989312:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829508.989315:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829508.989318:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989319:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829508.989322:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829508.989325:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989328:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829508.989332:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829508.989337:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829508.989342:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829508.989343:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829508.989345:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829508.989348:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829508.989351:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829508.989357:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829508.989362:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829508.989377:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327351552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829508.989389:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829508.989391:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829508.989397:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327351552/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829508.989404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829508.989406:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829508.989411:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1787829508.989412:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3666)
00000800:00000200:0.1:1787829508.989887:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829508.989891:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829508.990166:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829508.990169:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829508.990446:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829508.990449:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787829509.500755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829509.500761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829509.500762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829509.500767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829509.500770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829509.500774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829509.500776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829509.500778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829509.500780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829509.692727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298329772, at 4298329792
00000100:00000040:3.0:1787829509.692727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298329772, at 4298329792
00000100:00000010:2.0:1787829509.692742:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000100:00000010:3.0:1787829509.692745:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000020:00000001:2.0:1787829509.820863:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829509.820866:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829509.820869:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829510.524782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829510.524787:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829510.524789:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829510.524793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829510.524797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829510.524801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829510.524804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829510.524806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829510.524808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829510.844772:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829510.844775:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829510.844778:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829511.548773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829511.548780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829511.548782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829511.548787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829511.548790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829511.548793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829511.548796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829511.548798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829511.548800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829512.572796:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829512.572804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829512.572806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829512.572811:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829512.572815:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829512.572818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829512.572821:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829512.572823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829512.572827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829512.892843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829512.892847:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829512.892851:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829513.596745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829513.596753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829513.596755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829513.596760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829513.596763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829513.596767:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829513.596769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829513.596771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829513.596774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829513.916890:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829513.916897:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829513.916902:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829514.108801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829514.108810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829514.108812:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829514.108816:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829514.108830:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829514.108836:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3661
00000400:00000200:3.0:1787829514.108844:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829514.108849:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829514.108852:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829514.108854:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108856:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.108858:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108860:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829514.108863:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829514.108864:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829514.108866:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829514.108869:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829514.108871:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829514.108872:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829514.108874:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108875:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.108877:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108878:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829514.108880:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829514.108883:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108884:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829514.108886:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829514.108890:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829514.108892:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.108894:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829514.108896:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829514.108904:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829514.108907:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829514.108910:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829514.108912:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.108914:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829514.108916:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829514.108920:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829514.108926:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829514.108931:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829514.108938:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327351680/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829514.108946:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829514.108949:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829514.108952:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327351680/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829514.108975:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829514.108977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829514.108980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829514.108981:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829514.108982:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829514.108984:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829514.108985:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3661
00000400:00000200:3.0:1787829514.108988:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829514.108990:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829514.108991:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829514.108993:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108995:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.108997:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.108999:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829514.109000:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829514.109002:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829514.109003:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829514.109005:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829514.109007:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829514.109008:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829514.109010:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109012:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.109013:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109015:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829514.109019:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829514.109023:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109027:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829514.109029:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829514.109034:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829514.109040:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109045:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829514.109049:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829514.109055:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109059:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829514.109064:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829514.109067:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829514.109071:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829514.109074:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109079:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829514.109082:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829514.109086:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829514.109094:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829514.109099:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829514.109108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327351808/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829514.109122:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829514.109125:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829514.109130:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327351808/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829514.109141:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829514.109145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829514.109149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829514.109153:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829514.109155:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829514.109160:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829514.109166:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3661
00000400:00000200:3.0:1787829514.109171:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829514.109176:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829514.109180:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829514.109186:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109189:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.109197:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109200:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829514.109204:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829514.109205:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829514.109207:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829514.109210:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829514.109213:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829514.109216:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829514.109223:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109229:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.109237:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109242:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829514.109248:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829514.109256:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109263:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829514.109269:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829514.109274:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829514.109283:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109289:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829514.109293:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829514.109301:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109307:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829514.109314:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829514.109321:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829514.109327:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829514.109332:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829514.109332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829514.109337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:3.0:1787829514.109338:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829514.109342:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829514.109348:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829514.109355:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829514.109364:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829514.109376:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327351936/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829514.109402:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829514.109406:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829514.109414:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327351936/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829514.109433:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829514.109436:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829514.109439:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829514.109443:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829514.109445:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829514.109450:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829514.109456:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3661
00000400:00000200:3.0:1787829514.109462:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829514.109468:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829514.109474:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829514.109480:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109488:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.109495:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109503:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829514.109509:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829514.109514:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829514.109518:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829514.109522:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829514.109526:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829514.109529:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829514.109535:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109540:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829514.109544:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109548:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829514.109552:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829514.109559:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829514.109564:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829514.109574:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829514.109579:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829514.109586:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109591:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829514.109596:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829514.109602:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109607:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829514.109614:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829514.109618:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829514.109623:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829514.109626:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829514.109630:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829514.109634:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829514.109637:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829514.109642:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829514.109648:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829514.109706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327352064/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829514.109721:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829514.109726:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829514.109734:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327352064/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829514.109743:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829514.109749:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3671)
00000800:00000200:0.1:1787829514.109953:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829514.109955:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829514.110275:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829514.110277:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829514.110587:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829514.110589:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1787829514.620759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829514.620765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829514.620766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829514.620771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829514.620774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829514.620776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829514.620778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829514.620781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829514.620783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829514.812756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298334892, at 4298334912
00000100:00000040:3.0:1787829514.812758:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298334892, at 4298334912
00000100:00000010:2.0:1787829514.812775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000100:00000010:3.0:1787829514.812777:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000100:00000001:2.0:1787829515.644745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829515.644753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829515.644755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829515.644760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829515.644763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829515.644766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829515.644768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829515.644771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829515.644773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829515.964903:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829515.964908:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829515.964911:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829516.668752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829516.668759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829516.668760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829516.668764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829516.668767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829516.668769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829516.668771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829516.668773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829516.668775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829516.988790:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829516.988795:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829516.988811:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829517.692738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829517.692747:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829517.692750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829517.692757:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829517.692761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829517.692765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829517.692769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829517.692772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829517.692776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829518.716772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829518.716779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829518.716781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829518.716796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829518.716800:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829518.716804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829518.716806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829518.716809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829518.716811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829519.036908:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829519.036913:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829519.036918:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829519.228775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829519.228795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829519.228797:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829519.228800:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829519.228804:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829519.228810:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3666
00000400:00000200:3.0:1787829519.228818:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829519.228824:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829519.228831:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829519.228834:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228837:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.228839:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228840:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829519.228844:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829519.228845:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829519.228847:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829519.228851:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829519.228852:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829519.228854:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829519.228856:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228857:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.228859:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228860:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829519.228862:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829519.228865:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228867:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829519.228869:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829519.228872:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829519.228874:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.228875:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829519.228877:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829519.228885:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829519.228887:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829519.228891:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829519.228892:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.228895:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829519.228897:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829519.228900:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829519.228906:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829519.228911:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829519.228918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327352192/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829519.228926:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829519.228929:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829519.228933:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327352192/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829519.228954:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829519.228956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829519.228959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829519.228960:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829519.228961:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829519.228963:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829519.228964:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3666
00000400:00000200:3.0:1787829519.228966:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829519.228968:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829519.228970:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829519.228972:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228974:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.228975:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228976:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829519.228978:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829519.228979:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829519.228980:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829519.228981:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829519.228983:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829519.228985:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829519.228987:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228989:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.228991:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.228992:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829519.228994:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829519.228997:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229001:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829519.229005:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829519.229012:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829519.229015:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229017:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829519.229022:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829519.229026:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229029:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829519.229031:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829519.229036:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829519.229039:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829519.229042:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229047:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829519.229048:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829519.229050:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829519.229055:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829519.229059:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829519.229066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327352320/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829519.229075:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829519.229079:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829519.229085:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327352320/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829519.229094:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829519.229098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829519.229102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829519.229105:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829519.229107:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829519.229109:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829519.229111:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3666
00000400:00000200:3.0:1787829519.229117:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829519.229121:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829519.229123:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829519.229129:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229132:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.229137:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229139:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829519.229140:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829519.229142:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829519.229146:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829519.229148:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829519.229150:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829519.229152:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829519.229158:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229162:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.229167:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229169:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829519.229174:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829519.229180:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229184:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829519.229187:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829519.229191:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829519.229195:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229197:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829519.229201:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829519.229205:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229208:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829519.229215:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829519.229218:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829519.229222:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829519.229225:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229227:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829519.229228:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829519.229232:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829519.229238:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829519.229244:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829519.229251:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327352448/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829519.229264:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829519.229267:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829519.229273:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327352448/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829519.229281:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829519.229283:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829519.229286:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829519.229289:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829519.229292:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829519.229297:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829519.229301:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3666
00000400:00000200:3.0:1787829519.229309:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829519.229314:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829519.229318:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829519.229321:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229326:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.229327:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229329:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829519.229332:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829519.229334:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829519.229342:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829519.229344:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829519.229350:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829519.229351:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829519.229355:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229361:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829519.229365:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229368:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829519.229368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787829519.229373:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000200:0.1:1787829519.229376:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00080000:3.0:1787829519.229377:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829519.229383:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829519.229385:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829519.229387:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829519.229389:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229392:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829519.229396:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829519.229399:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229400:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829519.229403:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829519.229408:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829519.229412:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829519.229414:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829519.229419:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829519.229421:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829519.229424:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829519.229428:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829519.229433:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829519.229441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327352576/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829519.229449:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829519.229453:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829519.229459:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327352576/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829519.229478:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829519.229486:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3677)
00000800:00000200:0.1:1787829519.229847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829519.229849:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829519.230099:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829519.230101:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829519.230301:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829519.230304:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787829519.740736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829519.740741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829519.740742:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829519.740746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829519.740749:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829519.740752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829519.740754:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829519.740756:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829519.740758:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829519.932754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298340012, at 4298340032
00000100:00000040:3.0:1787829519.932754:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298340012, at 4298340032
00000100:00000010:2.0:1787829519.932769:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000100:00000010:3.0:1787829519.932771:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000020:00000001:3.0:1787829520.060802:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829520.060806:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829520.060809:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829520.764769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829520.764788:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829520.764790:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829520.764795:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829520.764797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829520.764800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829520.764802:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829520.764804:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829520.764807:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829521.788725:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829521.788734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829521.788737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829521.788744:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829521.788748:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829521.788752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829521.788756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829521.788759:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829521.788779:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829522.108885:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829522.108889:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829522.108891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829522.812744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829522.812749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829522.812751:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829522.812755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829522.812768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829522.812771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829522.812773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829522.812776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829522.812778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829523.132824:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829523.132828:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829523.132831:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829523.836791:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829523.836800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829523.836802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829523.836808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829523.836812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829523.836816:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829523.836819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829523.836822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829523.836825:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829524.348758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829524.348768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829524.348771:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829524.348774:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829524.348777:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829524.348783:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3672
00000400:00000200:3.0:1787829524.348790:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829524.348795:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829524.348798:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829524.348801:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348803:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.348805:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348806:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829524.348809:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829524.348810:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829524.348813:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829524.348815:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829524.348817:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829524.348818:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829524.348820:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348821:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.348823:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348824:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829524.348827:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829524.348829:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348831:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829524.348834:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829524.348837:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829524.348839:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.348840:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829524.348842:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829524.348850:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829524.348854:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829524.348859:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829524.348860:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.348863:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829524.348866:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829524.348870:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829524.348878:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829524.348886:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829524.348895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327352704/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829524.348905:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829524.348909:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829524.348912:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327352704/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829524.348938:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829524.348941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829524.348943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829524.348944:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829524.348945:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829524.348947:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829524.348949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3672
00000400:00000200:3.0:1787829524.348952:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829524.348954:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829524.348956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829524.348958:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348960:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.348963:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348965:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829524.348967:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829524.348968:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829524.348970:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829524.348972:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829524.348973:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829524.348975:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829524.348977:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348979:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.348981:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348984:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829524.348987:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829524.348992:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829524.348994:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829524.348997:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829524.349002:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829524.349005:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349008:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829524.349017:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829524.349022:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349028:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829524.349031:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829524.349035:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829524.349042:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829524.349045:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349048:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829524.349050:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829524.349052:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829524.349059:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829524.349063:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829524.349067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327352832/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829524.349076:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829524.349080:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829524.349085:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327352832/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829524.349098:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829524.349106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829524.349115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829524.349119:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829524.349124:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829524.349128:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829524.349136:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3672
00000400:00000200:3.0:1787829524.349145:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829524.349152:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829524.349157:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829524.349166:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349173:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.349178:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349185:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829524.349190:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829524.349193:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829524.349195:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829524.349199:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829524.349202:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829524.349207:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829524.349213:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349214:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.349217:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349220:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829524.349225:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829524.349230:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349237:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829524.349239:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829524.349243:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829524.349247:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349250:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829524.349254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829524.349258:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349262:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829524.349267:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829524.349271:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829524.349277:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829524.349279:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349284:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829524.349285:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829524.349287:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829524.349291:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829524.349296:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829524.349303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327352960/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829524.349313:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829524.349316:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829524.349320:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327352960/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829524.349339:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829524.349344:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829524.349345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829524.349349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829524.349352:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1787829524.349353:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00080000:3.0:1787829524.349355:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829524.349360:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829524.349367:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3672
00000400:00000200:3.0:1787829524.349376:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829524.349385:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829524.349390:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829524.349399:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349406:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.349412:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349417:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829524.349424:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829524.349427:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829524.349431:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829524.349436:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829524.349442:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829524.349448:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829524.349458:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349470:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829524.349475:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349481:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829524.349485:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829524.349489:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829524.349491:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829524.349492:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829524.349494:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829524.349498:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349501:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829524.349503:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829524.349508:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349510:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829524.349514:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829524.349519:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829524.349522:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829524.349524:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829524.349528:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829524.349529:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829524.349532:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829524.349540:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829524.349548:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829524.349559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327353088/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829524.349591:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829524.349594:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829524.349601:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327353088/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829524.349613:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829524.349619:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3682)
00000800:00000200:0.1:1787829524.349972:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829524.349974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829524.350283:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829524.350286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829524.350579:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829524.350581:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1787829524.860802:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829524.860810:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829524.860812:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829524.860816:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829524.860820:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829524.860823:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829524.860825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829524.860827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829524.860830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829525.052791:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298345132, at 4298345152
00000100:00000040:3.0:1787829525.052792:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298345132, at 4298345152
00000100:00000010:2.0:1787829525.052813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000010:3.0:1787829525.052817:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000020:00000001:2.0:1787829525.180929:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829525.180932:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829525.180935:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829525.884834:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829525.884842:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829525.884843:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829525.884849:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829525.884852:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829525.884855:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829525.884857:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829525.884860:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829525.884862:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829526.204848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829526.204853:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829526.204857:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829526.908771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829526.908779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829526.908780:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829526.908785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829526.908788:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829526.908792:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829526.908794:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829526.908797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829526.908799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829527.932788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829527.932796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829527.932799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829527.932804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829527.932807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829527.932810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829527.932812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829527.932814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829527.932816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829528.252867:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829528.252870:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829528.252873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829528.956773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829528.956780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829528.956782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829528.956787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829528.956790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829528.956793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829528.956795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829528.956797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829528.956799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829529.276804:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829529.276807:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829529.276811:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829529.468810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829529.468825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829529.468828:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829529.468833:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829529.468838:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829529.468855:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3677
00000400:00000200:3.0:1787829529.468866:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829529.468875:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829529.468879:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829529.468882:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.468887:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.468889:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.468892:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829529.468896:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829529.468898:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829529.468901:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829529.468905:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829529.468907:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829529.468910:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829529.468913:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.468916:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.468918:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.468920:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829529.468924:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829529.468928:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829529.468931:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829529.468934:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829529.468939:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829529.468942:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.468945:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829529.468949:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829529.468960:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829529.468964:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829529.468968:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829529.468970:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.468974:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829529.468977:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829529.468982:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829529.468992:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829529.469000:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829529.469012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327353216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829529.469030:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829529.469035:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829529.469041:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327353216/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829529.469067:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829529.469075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829529.469078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829529.469080:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829529.469082:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829529.469085:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829529.469088:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3677
00000400:00000200:3.0:1787829529.469093:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829529.469097:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829529.469100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829529.469104:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469108:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.469111:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469115:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829529.469120:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829529.469121:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829529.469124:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829529.469127:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829529.469131:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829529.469135:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829529.469142:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469146:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.469149:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469153:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829529.469157:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829529.469160:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469165:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829529.469167:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829529.469175:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829529.469179:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469183:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829529.469188:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829529.469191:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469194:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829529.469197:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829529.469199:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829529.469202:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829529.469205:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469207:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829529.469209:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829529.469211:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829529.469215:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829529.469221:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829529.469228:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327353344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829529.469235:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829529.469239:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829529.469242:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327353344/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829529.469248:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829529.469252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829529.469254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829529.469256:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829529.469257:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829529.469263:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829529.469268:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3677
00000400:00000200:3.0:1787829529.469273:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829529.469278:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829529.469281:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829529.469294:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469296:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.469300:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469302:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829529.469307:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829529.469308:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829529.469310:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829529.469311:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829529.469315:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829529.469319:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829529.469322:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469323:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.469324:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469328:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829529.469332:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829529.469334:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469337:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829529.469340:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829529.469344:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829529.469347:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469349:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829529.469352:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829529.469356:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469359:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829529.469364:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829529.469369:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829529.469373:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829529.469376:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469377:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829529.469378:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829529.469380:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829529.469385:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829529.469388:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829529.469392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327353472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829529.469410:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829529.469413:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829529.469417:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327353472/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829529.469430:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829529.469432:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829529.469433:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829529.469437:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00080000:3.0:1787829529.469438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829529.469441:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829529.469443:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829529.469447:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829529.469451:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3677
00000400:00000200:3.0:1787829529.469456:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829529.469463:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829529.469468:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829529.469474:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469478:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.469483:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469486:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829529.469490:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829529.469493:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829529.469497:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829529.469501:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829529.469506:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829529.469512:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829529.469518:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469524:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829529.469529:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469535:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829529.469541:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829529.469546:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829529.469552:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829529.469562:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829529.469567:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829529.469573:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469577:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829529.469581:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829529.469585:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469587:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829529.469595:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829529.469601:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829529.469604:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829529.469606:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829529.469608:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829529.469610:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829529.469614:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829529.469618:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829529.469624:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829529.469632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327353600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829529.469645:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829529.469649:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829529.469674:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327353600/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829529.469681:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829529.469685:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3687)
00000800:00000200:0.1:1787829529.469951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829529.469953:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829529.470146:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829529.470149:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829529.470362:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829529.470364:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787829529.980800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829529.980808:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829529.980810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829529.980815:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829529.980819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829529.980822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829529.980825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829529.980827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829529.980829:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829530.172796:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298350252, at 4298350272
00000100:00000040:3.0:1787829530.172796:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298350252, at 4298350272
00000100:00000010:3.0:1787829530.172812:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:2.0:1787829530.172814:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000100:00000001:2.0:1787829531.004799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829531.004807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829531.004808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829531.004817:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829531.004820:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829531.004824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829531.004827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829531.004829:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829531.004831:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829531.324871:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829531.324874:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829531.324877:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829532.028783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829532.028791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829532.028794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829532.028802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829532.028807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829532.028811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829532.028814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829532.028819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829532.028823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829532.348832:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829532.348835:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829532.348838:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829533.052751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829533.052756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829533.052758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829533.052762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829533.052765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829533.052768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829533.052770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829533.052771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829533.052773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829534.076810:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829534.076820:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829534.076823:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829534.076830:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829534.076834:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829534.076839:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829534.076842:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829534.076845:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829534.076848:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829534.396859:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829534.396864:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829534.396868:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829534.588789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829534.588799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829534.588801:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829534.588805:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829534.588809:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829534.588826:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3682
00000400:00000200:3.0:1787829534.588836:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829534.588844:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829534.588849:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829534.588853:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.588857:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.588858:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.588860:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829534.588864:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829534.588866:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829534.588868:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829534.588871:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829534.588873:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829534.588874:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829534.588876:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.588878:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.588879:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.588881:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829534.588883:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829534.588885:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829534.588888:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829534.588890:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829534.588893:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829534.588895:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.588897:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829534.588899:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829534.588910:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829534.588912:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829534.588916:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829534.588917:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.588920:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829534.588924:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829534.588927:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829534.588934:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829534.588940:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829534.588950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327353728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829534.588958:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829534.588961:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829534.588965:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327353728/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829534.588983:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829534.588985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829534.588988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829534.588989:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829534.588990:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829534.588992:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829534.588994:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3682
00000400:00000200:3.0:1787829534.588996:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829534.588998:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829534.588999:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829534.589001:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589003:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.589004:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589006:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829534.589007:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829534.589009:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829534.589009:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829534.589011:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829534.589012:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829534.589013:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829534.589015:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589016:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.589017:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589018:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829534.589020:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829534.589022:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589024:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829534.589025:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829534.589030:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829534.589032:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589033:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829534.589038:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829534.589041:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589042:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829534.589045:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829534.589047:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829534.589048:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829534.589049:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589050:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829534.589051:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829534.589052:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829534.589056:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829534.589060:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829534.589066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327353856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829534.589073:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829534.589077:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829534.589083:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327353856/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829534.589091:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829534.589093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829534.589097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829534.589099:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829534.589103:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829534.589111:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829534.589118:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3682
00000400:00000200:3.0:1787829534.589139:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829534.589148:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829534.589152:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829534.589159:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589164:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.589171:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589175:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829534.589182:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829534.589184:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829534.589188:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829534.589192:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829534.589195:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829534.589199:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829534.589204:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589214:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.589218:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589223:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829534.589225:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829534.589232:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589234:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829534.589235:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829534.589237:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829534.589239:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589241:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829534.589244:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829534.589248:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589253:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829534.589256:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829534.589261:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829534.589265:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829534.589266:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589269:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829534.589270:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829534.589272:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829534.589276:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829534.589280:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829534.589285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327353984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829534.589294:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829534.589296:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829534.589307:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327353984/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829534.589320:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829534.589324:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829534.589328:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829534.589331:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829534.589335:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829534.589340:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829534.589345:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3682
00000400:00000200:3.0:1787829534.589355:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829534.589364:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829534.589371:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829534.589381:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589389:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.589399:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589404:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829534.589407:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829534.589409:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829534.589410:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829534.589412:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829534.589415:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829534.589421:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829534.589426:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589428:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829534.589429:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589430:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829534.589432:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829534.589436:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829534.589441:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829534.589446:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829534.589448:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829534.589449:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589451:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829534.589452:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829534.589456:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589464:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829534.589472:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829534.589478:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829534.589483:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829534.589486:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829534.589488:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829534.589491:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829534.589494:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829534.589498:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829534.589505:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829534.589511:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327354112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829534.589522:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829534.589526:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829534.589537:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327354112/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829534.589550:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829534.589555:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3692)
00000800:00000200:0.1:1787829534.589602:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829534.589606:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829534.590140:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829534.590143:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829534.590469:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829534.590472:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829534.590836:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829534.590839:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1787829535.100751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829535.100759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829535.100762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829535.100766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829535.100769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829535.100772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829535.100774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829535.100776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829535.100779:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829535.292759:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298355373, at 4298355392
00000100:00000040:3.0:1787829535.292762:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298355373, at 4298355392
00000100:00000010:2.0:1787829535.292776:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00000010:3.0:1787829535.292782:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000020:00000001:3.0:1787829535.420858:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829535.420862:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829535.420865:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829536.124786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829536.124796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829536.124798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829536.124806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829536.124811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829536.124815:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829536.124819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829536.124823:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829536.124826:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829537.148749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829537.148757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829537.148759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829537.148764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829537.148767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829537.148770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829537.148773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829537.148775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829537.148777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829537.468843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829537.468848:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829537.468857:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829538.172802:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829538.172811:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829538.172813:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829538.172818:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829538.172821:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829538.172825:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829538.172828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829538.172831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829538.172833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829538.492864:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829538.492869:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829538.492873:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829539.196731:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829539.196743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829539.196746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829539.196754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829539.196760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829539.196763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829539.196767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829539.196770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829539.196782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829539.708800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829539.708807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829539.708809:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829539.708812:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829539.708815:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829539.708821:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3687
00000400:00000200:1.0:1787829539.708828:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829539.708833:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829539.708835:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829539.708838:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708840:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.708842:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708844:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829539.708847:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829539.708849:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829539.708851:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829539.708855:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829539.708856:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829539.708858:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829539.708860:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708862:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.708863:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708864:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829539.708867:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829539.708871:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708874:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829539.708876:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829539.708879:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829539.708881:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.708883:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829539.708885:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829539.708894:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829539.708897:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829539.708899:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829539.708901:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.708903:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829539.708905:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829539.708908:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829539.708914:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829539.708919:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829539.708926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327354240/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829539.708934:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829539.708937:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829539.708941:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327354240/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829539.708959:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829539.708961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829539.708964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829539.708965:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829539.708966:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829539.708968:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829539.708969:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3687
00000400:00000200:1.0:1787829539.708972:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829539.708974:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829539.708975:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829539.708978:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708980:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.708981:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708983:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829539.708985:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829539.708986:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829539.708987:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829539.708988:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829539.708989:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829539.708991:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829539.708993:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.708994:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.708997:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709001:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829539.709003:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829539.709005:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709011:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829539.709014:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829539.709019:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829539.709023:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709029:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829539.709034:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829539.709040:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709045:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829539.709051:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829539.709056:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829539.709065:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829539.709077:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709082:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829539.709086:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829539.709091:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829539.709100:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829539.709106:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829539.709117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327354368/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829539.709138:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829539.709143:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829539.709150:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327354368/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829539.709171:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829539.709176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829539.709184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829539.709186:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829539.709190:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829539.709196:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829539.709201:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3687
00000400:00000200:1.0:1787829539.709206:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829539.709211:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829539.709216:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829539.709218:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709220:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.709225:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709230:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829539.709232:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829539.709233:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829539.709234:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829539.709236:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829539.709238:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829539.709241:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829539.709247:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709248:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.709255:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709260:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829539.709272:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829539.709285:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709291:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829539.709297:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829539.709303:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829539.709309:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709320:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829539.709322:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829539.709324:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709325:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829539.709328:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829539.709331:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829539.709336:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829539.709339:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709344:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829539.709348:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829539.709352:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829539.709360:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829539.709366:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829539.709375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327354496/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829539.709396:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829539.709397:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829539.709401:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829539.709405:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:1.0:1787829539.709408:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327354496/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829539.709420:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829539.709424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829539.709428:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829539.709430:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829539.709433:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829539.709437:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829539.709443:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3687
00000400:00000200:1.0:1787829539.709452:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829539.709460:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829539.709464:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829539.709470:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709475:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.709481:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709487:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829539.709492:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829539.709495:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829539.709498:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829539.709501:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829539.709506:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829539.709511:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829539.709521:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709528:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829539.709533:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709538:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829539.709541:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829539.709545:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829539.709549:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829539.709554:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829539.709559:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829539.709563:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709567:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829539.709569:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829539.709571:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709575:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829539.709579:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829539.709582:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829539.709586:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829539.709588:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829539.709590:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829539.709592:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829539.709595:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829539.709602:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829539.709608:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829539.709615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327354624/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829539.709621:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829539.709622:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829539.709625:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327354624/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829539.709690:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829539.709699:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3697)
00000800:00000200:0.1:1787829539.709992:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829539.709993:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829539.710239:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829539.710241:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829539.710558:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829539.710560:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787829540.220750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829540.220757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829540.220759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829540.220765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829540.220768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829540.220771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829540.220773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829540.220775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829540.220778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829540.412766:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298360492, at 4298360512
00000100:00000040:3.0:1787829540.412766:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298360492, at 4298360512
00000100:00000010:2.0:1787829540.412785:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000100:00000010:3.0:1787829540.412787:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000020:00000001:2.0:1787829540.540889:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829540.540894:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829540.540898:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829541.244788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829541.244796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829541.244797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829541.244803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829541.244807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829541.244810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829541.244813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829541.244815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829541.244817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829541.564842:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829541.564847:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829541.564851:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829542.268805:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829542.268816:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829542.268819:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829542.268826:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829542.268831:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829542.268836:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829542.268839:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829542.268843:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829542.268846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829543.292747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829543.292756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829543.292759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829543.292765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829543.292769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829543.292773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829543.292775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829543.292778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829543.292781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829543.612804:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829543.612807:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829543.612810:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829544.316747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829544.316754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829544.316756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829544.316761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829544.316764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829544.316768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829544.316770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829544.316773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829544.316775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829544.636856:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829544.636860:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829544.636863:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829544.828755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829544.828766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829544.828768:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829544.828772:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829544.828777:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829544.828784:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3692
00000400:00000200:1.0:1787829544.828794:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829544.828800:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829544.828803:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829544.828806:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.828809:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.828811:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.828813:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829544.828816:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829544.828817:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829544.828820:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829544.828823:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829544.828824:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829544.828827:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829544.828829:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.828830:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.828831:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.828833:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829544.828835:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829544.828838:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829544.828839:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829544.828842:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829544.828845:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829544.828847:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.828849:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829544.828851:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829544.828861:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829544.828864:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829544.828868:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829544.828870:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.828873:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829544.828875:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829544.828880:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829544.828888:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829544.828897:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829544.828910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327354752/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829544.828923:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829544.828929:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829544.828935:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327354752/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829544.828962:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829544.828965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829544.828969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829544.828970:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829544.828972:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829544.828975:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829544.828978:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3692
00000400:00000200:1.0:1787829544.828982:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829544.828985:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829544.828988:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829544.828992:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.828995:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.828997:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829000:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829544.829002:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829544.829004:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829544.829009:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829544.829011:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829544.829015:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829544.829019:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829544.829024:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829027:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.829031:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829035:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829544.829041:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829544.829047:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829051:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829544.829053:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829544.829059:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829544.829064:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829069:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829544.829075:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829544.829082:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829087:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829544.829095:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829544.829100:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829544.829106:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829544.829109:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829113:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829544.829117:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829544.829120:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829544.829128:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829544.829134:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829544.829145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327354880/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829544.829161:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829544.829166:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829544.829172:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327354880/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829544.829184:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829544.829189:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829544.829198:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829544.829203:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829544.829206:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829544.829213:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829544.829222:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3692
00000400:00000200:1.0:1787829544.829233:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829544.829255:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829544.829260:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829544.829266:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829269:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.829271:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829273:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829544.829276:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829544.829278:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829544.829281:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829544.829284:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829544.829287:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829544.829291:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829544.829296:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829299:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.829304:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829307:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829544.829312:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829544.829319:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829324:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829544.829328:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829544.829332:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829544.829335:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829337:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829544.829339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829544.829342:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829345:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829544.829351:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829544.829357:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829544.829360:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829544.829361:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829364:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829544.829366:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829544.829369:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829544.829375:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829544.829379:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829544.829385:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327355008/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829544.829394:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829544.829407:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829544.829410:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829544.829416:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1787829544.829417:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327355008/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829544.829435:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829544.829439:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829544.829449:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829544.829452:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829544.829456:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829544.829462:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829544.829470:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3692
00000400:00000200:1.0:1787829544.829480:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829544.829487:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829544.829494:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829544.829502:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829505:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.829507:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829509:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829544.829514:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829544.829516:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829544.829520:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829544.829524:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829544.829529:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829544.829532:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829544.829536:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829539:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829544.829540:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829542:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829544.829544:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829544.829548:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829544.829550:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829544.829552:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829544.829555:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829544.829559:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829562:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829544.829567:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829544.829570:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829573:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829544.829580:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829544.829585:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829544.829589:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829544.829590:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829544.829592:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829544.829592:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829544.829595:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829544.829601:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829544.829618:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829544.829625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327355136/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829544.829638:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829544.829641:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829544.829648:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327355136/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829544.829677:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829544.829681:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3702)
00000800:00000200:0.1:1787829544.829941:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829544.829942:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829544.830256:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829544.830258:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829544.830496:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829544.830498:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787829545.340768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829545.340773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829545.340775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829545.340779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829545.340782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829545.340786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829545.340788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829545.340790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829545.340792:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829545.532762:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298365612, at 4298365632
00000100:00000040:3.0:1787829545.532762:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298365612, at 4298365632
00000100:00000010:3.0:1787829545.532777:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000100:00000010:2.0:1787829545.532780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000001:2.0:1787829546.364773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829546.364783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829546.364785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829546.364792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829546.364796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829546.364800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829546.364803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829546.364809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829546.364813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829546.684940:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829546.684945:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829546.684949:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829547.388755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829547.388760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829547.388762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829547.388766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829547.388769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829547.388772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829547.388774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829547.388776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829547.388778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829547.708833:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829547.708838:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829547.708842:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829548.412747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829548.412753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829548.412755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829548.412760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829548.412762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829548.412765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829548.412767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829548.412769:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829548.412771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829549.436779:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829549.436785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829549.436788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829549.436793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829549.436797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829549.436801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829549.436804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829549.436807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829549.436811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829549.756806:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829549.756809:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829549.756812:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829549.948698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829549.948707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829549.948710:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829549.948713:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829549.948717:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829549.948724:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3697
00000400:00000200:1.0:1787829549.948733:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829549.948738:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829549.948741:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829549.948744:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948746:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.948749:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948751:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829549.948754:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829549.948755:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829549.948757:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829549.948760:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829549.948762:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829549.948764:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829549.948767:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948769:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.948771:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948773:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829549.948776:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829549.948778:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948781:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829549.948783:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829549.948786:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829549.948788:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.948790:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829549.948792:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829549.948799:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829549.948802:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829549.948806:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829549.948807:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.948810:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829549.948812:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829549.948815:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829549.948822:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829549.948828:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829549.948835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327355264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829549.948844:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829549.948848:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829549.948852:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327355264/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829549.948872:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829549.948875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829549.948878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829549.948879:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829549.948880:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829549.948882:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829549.948885:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3697
00000400:00000200:1.0:1787829549.948889:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829549.948894:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829549.948897:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829549.948902:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948906:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.948908:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948910:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829549.948912:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829549.948913:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829549.948915:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829549.948919:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829549.948922:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829549.948926:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829549.948933:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948936:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.948940:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948944:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829549.948948:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829549.948952:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829549.948957:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829549.948959:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829549.948965:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829549.948969:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.948971:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829549.948976:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829549.948980:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.948983:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829549.948986:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829549.948990:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829549.948992:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829549.948995:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.948998:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829549.949002:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829549.949003:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829549.949007:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829549.949018:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829549.949028:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327355392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829549.949038:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829549.949040:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829549.949043:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327355392/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829549.949053:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829549.949058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829549.949062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829549.949065:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829549.949068:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829549.949071:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829549.949076:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3697
00000400:00000200:1.0:1787829549.949080:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829549.949085:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829549.949088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829549.949094:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949099:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.949103:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949107:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829549.949110:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829549.949113:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829549.949117:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829549.949118:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829549.949120:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829549.949126:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829549.949129:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949132:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.949136:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949138:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829549.949139:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829549.949144:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949151:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829549.949154:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829549.949158:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829549.949163:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.949164:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829549.949166:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829549.949170:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.949174:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829549.949179:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829549.949182:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829549.949186:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829549.949187:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.949189:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829549.949191:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829549.949194:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829549.949198:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829549.949201:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829549.949208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327355520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829549.949219:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829549.949223:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829549.949229:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327355520/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829549.949236:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829549.949240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829549.949245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829549.949247:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829549.949249:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829549.949254:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829549.949259:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3697
00000400:00000200:1.0:1787829549.949265:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829549.949270:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829549.949272:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829549.949277:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949280:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.949282:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949284:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829549.949288:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829549.949289:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829549.949290:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829549.949292:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829549.949295:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829549.949298:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829549.949305:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949307:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829549.949312:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949316:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829549.949318:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:1.0:1787829549.949321:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000200:0.1:1787829549.949325:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00080000:1.0:1787829549.949329:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829549.949337:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829549.949343:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829549.949348:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829549.949353:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.949357:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829549.949362:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829549.949366:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.949368:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829549.949375:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829549.949379:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829549.949382:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829549.949385:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829549.949387:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829549.949390:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829549.949392:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829549.949398:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829549.949403:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829549.949412:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327355648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829549.949422:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829549.949426:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829549.949434:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327355648/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829549.949447:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829549.949453:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3707)
00000800:00000200:0.1:1787829549.949884:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829549.949887:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829549.950190:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829549.950192:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829549.950495:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829549.950497:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1787829550.460768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829550.460773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829550.460775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829550.460779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829550.460782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829550.460784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829550.460786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829550.460788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829550.460790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829550.652711:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298370732, at 4298370752
00000100:00000040:3.0:1787829550.652714:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298370732, at 4298370752
00000100:00000010:2.0:1787829550.652726:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:3.0:1787829550.652728:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000020:00000001:3.0:1787829550.780816:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829550.780820:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829550.780824:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829551.484788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829551.484796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829551.484798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829551.484804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829551.484808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829551.484812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829551.484815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829551.484817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829551.484820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829552.508753:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829552.508763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829552.508766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829552.508773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829552.508778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829552.508782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829552.508786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829552.508789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829552.508793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829552.828846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829552.828851:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829552.828854:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829553.532755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829553.532766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829553.532769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829553.532776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829553.532781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829553.532786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829553.532788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829553.532792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829553.532800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829553.852833:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829553.852838:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829553.852843:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829554.556757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829554.556769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829554.556771:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829554.556777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829554.556781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829554.556785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829554.556787:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829554.556790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829554.556793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829555.068803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829555.068815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829555.068818:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829555.068823:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829555.068828:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829555.068835:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3702
00000400:00000200:1.0:1787829555.068845:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829555.068852:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829555.068855:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829555.068858:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.068861:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.068864:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.068866:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829555.068870:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829555.068872:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829555.068874:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829555.068878:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829555.068880:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829555.068881:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829555.068883:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.068885:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.068886:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.068887:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829555.068890:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829555.068893:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829555.068896:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829555.068898:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829555.068902:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829555.068904:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.068906:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829555.068908:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829555.068920:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829555.068923:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829555.068927:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829555.068928:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.068931:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829555.068934:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829555.068939:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829555.068946:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829555.068952:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829555.068962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327355776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829555.068971:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829555.068976:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829555.068988:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327355776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829555.069008:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829555.069011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829555.069013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829555.069014:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829555.069016:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829555.069018:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829555.069020:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3702
00000400:00000200:1.0:1787829555.069022:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829555.069024:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829555.069025:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829555.069027:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069029:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.069031:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069033:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829555.069034:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829555.069035:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829555.069036:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829555.069038:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829555.069040:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829555.069041:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829555.069044:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069046:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.069047:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069049:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829555.069051:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829555.069055:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069061:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829555.069064:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829555.069070:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829555.069075:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069080:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829555.069087:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829555.069091:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069095:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829555.069100:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829555.069105:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829555.069110:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829555.069113:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069117:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829555.069120:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829555.069124:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829555.069127:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829555.069133:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829555.069140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327355904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829555.069150:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829555.069154:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829555.069161:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327355904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829555.069169:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829555.069172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829555.069175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829555.069177:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829555.069182:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829555.069187:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829555.069192:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3702
00000400:00000200:1.0:1787829555.069199:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829555.069207:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829555.069210:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829555.069217:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069222:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.069226:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069228:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829555.069231:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829555.069233:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829555.069234:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829555.069236:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829555.069240:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829555.069244:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829555.069247:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069248:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.069250:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069254:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829555.069257:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829555.069260:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069264:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829555.069267:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829555.069270:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829555.069274:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069278:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829555.069281:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829555.069284:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069286:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829555.069291:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829555.069296:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829555.069300:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829555.069303:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069306:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829555.069309:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829555.069310:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829555.069314:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829555.069319:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829555.069327:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327356032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829555.069339:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829555.069342:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829555.069347:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327356032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829555.069359:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829555.069361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829555.069363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829555.069364:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829555.069365:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829555.069370:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1787829555.069370:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829555.069375:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3702
00000400:00000200:1.0:1787829555.069379:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829555.069381:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829555.069383:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829555.069385:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069386:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.069388:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069389:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829555.069390:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829555.069391:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829555.069392:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829555.069395:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829555.069400:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829555.069403:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829555.069410:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069414:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829555.069416:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069421:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829555.069423:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829555.069426:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829555.069431:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829555.069432:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829555.069435:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829555.069438:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069441:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829555.069443:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829555.069447:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069449:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829555.069455:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829555.069460:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829555.069466:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829555.069469:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829555.069472:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829555.069475:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829555.069476:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829555.069481:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829555.069486:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829555.069494:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327356160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829555.069506:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829555.069509:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829555.069518:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327356160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829555.069548:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829555.069554:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3712)
00000800:00000200:0.1:1787829555.069695:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829555.069855:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829555.070018:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829555.580790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829555.580798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829555.580800:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829555.580805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829555.580809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829555.580813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829555.580815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829555.580818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829555.580820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829555.772708:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298375852, at 4298375872
00000100:00000040:3.0:1787829555.772708:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298375852, at 4298375872
00000100:00000010:2.0:1787829555.772723:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00000100:00000010:3.0:1787829555.772725:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000020:00000001:2.0:1787829555.900868:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829555.900875:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829555.900880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829556.604712:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829556.604719:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829556.604721:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829556.604725:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829556.604728:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829556.604730:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829556.604733:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829556.604735:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829556.604737:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829556.924831:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829556.924836:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829556.924841:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829557.628762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829557.628768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829557.628769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829557.628773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829557.628776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829557.628779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829557.628781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829557.628784:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829557.628788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1787829558.140981:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829558.141028:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829558.141039:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829558.141044:0:2335:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829558.141049:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829558.141053:0:2335:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829558.141059:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829558.141064:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:2.0:1787829558.141070:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829558.141075:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829558.141081:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef220
00000400:00000010:2.0:1787829558.141084:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef220.
00000100:00000001:2.0:1787829558.141090:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829558.141098:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e21c0 x1874671327355776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141123:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829558.141128:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327355776/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141136:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829558.141138:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829558.141148:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829558.141151:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:2.0:1787829558.141155:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829558.141158:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829558.141161:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:2.0:1787829558.141163:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:2.0:1787829558.141166:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829558.141171:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e3c00 x1874671327355904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141179:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829558.141182:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327355904/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141189:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829558.141190:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829558.141195:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829558.141198:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000200:2.0:1787829558.141201:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829558.141207:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829558.141210:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefee0
00000400:00000010:2.0:1787829558.141212:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefee0.
00000100:00000001:2.0:1787829558.141216:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829558.141220:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e0000 x1874671327356032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141233:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829558.141241:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327356032/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141257:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829558.141262:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829558.141273:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829558.141280:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000200:2.0:1787829558.141286:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829558.141295:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829558.141300:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:2.0:1787829558.141305:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:2.0:1787829558.141311:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829558.141321:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e12c0 x1874671327356160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141354:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829558.141362:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327356160/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829558.141378:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829558.141383:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829558.652756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829558.652765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829558.652768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829558.652774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829558.652778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829558.652783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829558.652786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829558.652789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829558.652793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829558.972874:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829558.972877:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829558.972880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829559.676798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829559.676809:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829559.676812:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829559.676820:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829559.676825:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829559.676830:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829559.676833:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829559.676837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829559.676840:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829559.996831:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829559.996836:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829559.996841:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829560.188773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829560.188785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829560.188788:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829560.188793:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829560.188798:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829560.188806:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3707
00000400:00000200:3.0:1787829560.188816:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829560.188822:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829560.188825:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829560.188827:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.188830:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.188832:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.188835:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829560.188838:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829560.188840:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829560.188843:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829560.188847:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829560.188848:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829560.188850:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829560.188852:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.188853:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.188854:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.188856:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829560.188858:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829560.188861:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829560.188863:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829560.188865:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829560.188869:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829560.188871:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.188873:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829560.188875:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829560.188881:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829560.188884:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829560.188887:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829560.188889:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.188892:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829560.188895:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829560.188900:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829560.188907:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829560.188913:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829560.188924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327356288/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829560.188936:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829560.188942:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829560.188948:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327356288/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829560.188976:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829560.188980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829560.188983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829560.188985:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829560.188987:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829560.188990:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829560.188992:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3707
00000400:00000200:3.0:1787829560.188997:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829560.189001:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829560.189004:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829560.189007:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189010:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.189012:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189014:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829560.189017:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829560.189018:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829560.189020:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829560.189023:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829560.189025:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829560.189027:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829560.189031:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189033:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.189035:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189037:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829560.189040:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829560.189043:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189046:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829560.189048:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829560.189053:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829560.189055:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189057:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829560.189061:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829560.189063:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189070:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829560.189073:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829560.189075:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829560.189077:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829560.189078:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189080:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829560.189082:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829560.189084:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829560.189089:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829560.189094:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829560.189100:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327356416/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829560.189114:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829560.189117:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829560.189121:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327356416/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829560.189129:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829560.189132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829560.189136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829560.189138:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829560.189139:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829560.189142:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829560.189145:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3707
00000400:00000200:3.0:1787829560.189149:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829560.189153:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829560.189155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829560.189159:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189161:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.189164:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189166:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829560.189169:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829560.189170:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829560.189172:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829560.189175:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829560.189177:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829560.189179:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829560.189183:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189185:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.189187:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189190:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829560.189192:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829560.189195:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189198:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829560.189200:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829560.189203:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829560.189206:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189208:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829560.189211:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829560.189214:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189216:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829560.189221:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829560.189225:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829560.189228:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829560.189229:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189231:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829560.189233:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829560.189235:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829560.189240:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829560.189245:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829560.189251:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327356544/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829560.189260:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829560.189263:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829560.189268:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327356544/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829560.189278:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829560.189283:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829560.189288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829560.189289:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829560.189291:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829560.189295:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829560.189298:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3707
00000400:00000200:3.0:1787829560.189303:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829560.189309:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829560.189314:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829560.189323:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189331:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.189338:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189342:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829560.189345:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829560.189346:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829560.189348:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829560.189353:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829560.189358:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829560.189369:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829560.189383:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189394:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829560.189402:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189409:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829560.189417:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829560.189430:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829560.189442:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829560.189448:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829560.189454:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829560.189460:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189466:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829560.189473:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829560.189485:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189491:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829560.189598:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829560.189606:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829560.189614:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829560.189619:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829560.189626:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829560.189633:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829560.189639:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829560.189716:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829560.189730:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829560.189747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327356672/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829560.189765:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829560.189772:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829560.189784:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327356672/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829560.189790:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829560.189804:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:3.0:1787829560.189814:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829560.189829:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3717)
00000800:00000200:0.1:1787829560.190231:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829560.190233:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829560.190481:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829560.190483:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829560.190806:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829560.190809:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1787829560.700756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829560.700762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829560.700764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829560.700768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829560.700772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829560.700774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829560.700777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829560.700779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829560.700781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829560.892719:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298380973, at 4298380992
00000100:00000040:3.0:1787829560.892719:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298380973, at 4298380992
00000100:00000010:2.0:1787829560.892733:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:3.0:1787829560.892739:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000001:2.0:1787829561.724778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829561.724786:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829561.724788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829561.724799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829561.724803:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829561.724807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829561.724810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829561.724812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829561.724815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829562.044885:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829562.044891:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829562.044897:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829562.748785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829562.748793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829562.748795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829562.748800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829562.748804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829562.748808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829562.748811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829562.748813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829562.748816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829563.068861:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829563.068867:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829563.068870:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829563.772799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829563.772809:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829563.772812:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829563.772828:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829563.772833:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829563.772838:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829563.772841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829563.772844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829563.772848:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829564.796758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829564.796766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829564.796769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829564.796774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829564.796778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829564.796781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829564.796784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829564.796787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829564.796789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829565.116875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829565.116880:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829565.116883:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829565.308729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829565.308737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829565.308739:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829565.308743:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829565.308746:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829565.308752:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3712
00000400:00000200:3.0:1787829565.308759:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829565.308765:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829565.308767:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829565.308769:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308772:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.308773:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308775:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829565.308779:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829565.308780:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829565.308782:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829565.308785:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829565.308786:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829565.308788:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829565.308790:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308791:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.308793:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308794:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829565.308796:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829565.308799:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308800:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829565.308802:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829565.308805:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829565.308807:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.308809:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829565.308811:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829565.308818:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829565.308821:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829565.308824:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829565.308825:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.308828:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829565.308830:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829565.308834:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829565.308839:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829565.308844:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829565.308852:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327356800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829565.308860:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829565.308864:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829565.308867:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327356800/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829565.308894:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829565.308897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829565.308899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829565.308901:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829565.308902:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829565.308904:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829565.308905:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3712
00000400:00000200:3.0:1787829565.308908:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829565.308909:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829565.308911:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829565.308913:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308915:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.308917:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308919:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829565.308921:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829565.308923:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829565.308924:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829565.308925:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829565.308927:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829565.308929:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829565.308931:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308932:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.308934:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308938:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829565.308940:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829565.308945:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829565.308948:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829565.308951:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829565.308956:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829565.308962:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.308967:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829565.308972:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829565.308979:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.308982:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829565.308987:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829565.308993:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829565.308996:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829565.308999:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.309002:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829565.309005:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829565.309007:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829565.309015:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829565.309023:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829565.309035:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327356928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829565.309049:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829565.309057:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829565.309068:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327356928/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829565.309092:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829565.309099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829565.309105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829565.309111:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829565.309114:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829565.309116:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829565.309119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3712
00000400:00000200:3.0:1787829565.309125:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829565.309132:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829565.309137:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829565.309145:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309149:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.309155:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309159:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829565.309163:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829565.309166:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829565.309169:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829565.309173:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829565.309176:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829565.309178:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829565.309184:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309190:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.309196:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309199:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829565.309202:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829565.309207:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309212:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829565.309215:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829565.309219:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829565.309222:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.309227:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829565.309231:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.1:1787829565.309231:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:3.0:1787829565.309237:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829565.309237:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000100:00000001:3.0:1787829565.309243:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829565.309250:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829565.309253:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829565.309255:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829565.309258:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.309261:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829565.309262:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829565.309264:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829565.309271:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829565.309278:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829565.309287:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327357056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829565.309301:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829565.309305:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829565.309313:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327357056/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829565.309334:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829565.309338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829565.309343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829565.309345:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829565.309347:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829565.309349:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829565.309354:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3712
00000400:00000200:3.0:1787829565.309360:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829565.309367:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829565.309369:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829565.309374:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309376:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.309378:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309380:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829565.309383:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829565.309384:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829565.309386:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829565.309389:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829565.309393:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829565.309396:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829565.309402:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309406:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829565.309411:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309414:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829565.309417:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829565.309421:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829565.309427:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829565.309430:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829565.309434:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829565.309437:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.309440:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829565.309444:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829565.309448:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.309452:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829565.309458:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829565.309475:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829565.309478:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829565.309481:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829565.309485:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829565.309488:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829565.309491:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829565.309496:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829565.309502:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829565.309513:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327357184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829565.309525:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829565.309528:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829565.309535:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327357184/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829565.309550:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829565.309556:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3723)
00000800:00000200:0.1:1787829565.309746:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829565.309748:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787829565.309961:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829565.309963:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787829565.310142:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829565.310144:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 52 secs later)
00000100:00000001:2.0:1787829565.820810:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829565.820816:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829565.820818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829565.820821:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829565.820823:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829565.820826:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829565.820828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829565.820830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829565.820832:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829566.012740:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298386092, at 4298386112
00000100:00000040:3.0:1787829566.012742:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298386092, at 4298386112
00000100:00000010:2.0:1787829566.012756:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000100:00000010:3.0:1787829566.012758:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000020:00000001:3.0:1787829566.140827:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829566.140832:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829566.140836:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829566.844795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829566.844809:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829566.844811:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829566.844817:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829566.844820:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829566.844824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829566.844826:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829566.844829:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829566.844831:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829567.868749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829567.868757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829567.868760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829567.868766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829567.868770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829567.868775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829567.868778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829567.868782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829567.868786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829568.188893:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829568.188898:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829568.188901:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829568.892794:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829568.892801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829568.892804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829568.892809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829568.892812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829568.892816:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829568.892819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829568.892821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829568.892824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829569.212835:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829569.212841:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829569.212846:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829569.916786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829569.916795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829569.916797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829569.916802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829569.916805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829569.916809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829569.916811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829569.916814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829569.916816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829570.428751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829570.428760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829570.428762:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829570.428765:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829570.428769:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829570.428775:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3718
00000400:00000200:3.0:1787829570.428782:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829570.428787:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829570.428790:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829570.428792:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428794:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829570.428797:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428799:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829570.428802:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829570.428804:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829570.428806:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829570.428809:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829570.428811:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829570.428813:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829570.428815:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428816:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829570.428817:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428819:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829570.428821:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829570.428823:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428825:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829570.428827:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829570.428830:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829570.428832:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.428834:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829570.428835:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829570.428842:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829570.428845:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829570.428847:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829570.428848:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.428851:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829570.428853:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829570.428856:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829570.428862:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829570.428867:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829570.428874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829570.428882:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829570.428886:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829570.428890:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829570.428911:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829570.428914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829570.428916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829570.428917:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829570.428918:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829570.428920:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829570.428921:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3718
00000400:00000200:3.0:1787829570.428924:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829570.428926:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829570.428927:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829570.428929:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428931:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829570.428932:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428934:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829570.428935:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829570.428936:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829570.428938:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829570.428939:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829570.428941:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829570.428943:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829570.428945:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.428946:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829570.428948:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829570.428949:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787829570.428950:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829570.428954:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:1.0:1787829570.428955:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787829570.428957:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829570.428958:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829570.428963:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829570.428966:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787829570.428968:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829570.428971:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829570.428977:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.428981:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829570.428983:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829570.428986:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787829570.428986:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829570.428992:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1787829570.428992:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.428996:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.428997:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829570.429001:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829570.429002:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829570.429005:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829570.429007:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829570.429008:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.429009:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829570.429010:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:1.0:1787829570.429013:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327357312:192.168.203.116@tcp:250:kworker.0
02000000:00000001:3.0:1787829570.429013:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829570.429019:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:1.0:1787829570.429020:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829570.429024:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.429026:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829570.429029:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.429032:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1787829570.429032:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1787829570.429039:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:3.0:1787829570.429040:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787829570.429043:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.429044:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829570.429050:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829570.429053:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829570.429059:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000100:00000001:3.0:1787829570.429062:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829570.429065:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00080000:3.0:1787829570.429065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829570.429070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829570.429072:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829570.429073:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:1.0:1787829570.429075:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327357312, portal 25
00000100:00000001:3.0:1787829570.429076:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829570.429079:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3718
00000100:00000001:1.0:1787829570.429083:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829570.429086:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000400:00000200:3.0:1787829570.429087:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829570.429093:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829570.429097:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829570.429098:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829570.429101:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829570.429110:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787829570.429112:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829570.429117:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829570.429119:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829570.429120:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829570.429124:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829570.429124:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829570.429125:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787829570.429126:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000020:00000040:3.0:1787829570.429127:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829570.429132:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:1.0:1787829570.429134:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327357312, offset 0
00000020:00000001:3.0:1787829570.429135:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829570.429140:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.429143:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829570.429145:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829570.429146:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829570.429150:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:1.0:1787829570.429151:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829570.429155:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829570.429158:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829570.429159:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829570.429162:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829570.429162:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829570.429168:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.429170:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829570.429172:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787829570.429175:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000008:00000001:3.0:1787829570.429175:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.429179:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429185:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:3.0:1787829570.429185:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829570.429191:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:1.0:1787829570.429194:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:3.0:1787829570.429194:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829570.429197:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.429199:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787829570.429202:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787829570.429203:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829570.429205:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429210:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000010:3.0:1787829570.429210:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829570.429213:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429221:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40563:40563:256:4294967295] 192.168.203.116@tcp LPNI seq info [40563:40563:8:4294967295]
00000100:00100000:3.0:1787829570.429222:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829570.429233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829570.429240:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829570.429246:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429248:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787829570.429256:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1787829570.429257:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.429264:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00000001:3.0:1787829570.429272:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829570.429273:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829570.429278:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00080000:3.0:1787829570.429278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829570.429288:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:1.1:1787829570.429288:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829570.429294:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.1:1787829570.429295:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00080000:3.0:1787829570.429298:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000100:1.0:1787829570.429303:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829570.429305:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:1.0:1787829570.429306:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829570.429313:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1787829570.429313:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3718
00000400:00000200:1.0:1787829570.429323:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787829570.429324:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829570.429330:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829570.429333:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787829570.429334:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000100:00000040:3.0:1787829570.429337:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829570.429338:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:3.0:1787829570.429343:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829570.429345:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787829570.429349:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:1.0:1787829570.429352:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829570.429354:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829570.429360:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829570.429361:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829570.429365:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829570.429368:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829570.429371:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:3.0:1787829570.429373:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829570.429376:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829570.429379:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:1.0:1787829570.429387:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829570.429389:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829570.429390:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787829570.429395:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829570.429397:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829570.429398:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829570.429403:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829570.429403:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829570.429409:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298391212
00000020:00000040:3.0:1787829570.429410:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829570.429415:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000010:1.0:1787829570.429417:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63f60.
00000100:00000001:3.0:1787829570.429418:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829570.429423:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829570.429426:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829570.429441:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:3.0:1787829570.429442:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829570.429447:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829570.429450:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298391212
00000008:00100000:3.0:1787829570.429450:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829570.429457:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829570.429459:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63ae0.
00000100:00000001:3.0:1787829570.429460:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.429467:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000010:3.0:1787829570.429468:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829570.429474:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:1.0:1787829570.429475:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829570.429479:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829570.429480:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829570.429483:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.429484:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1787829570.429489:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1787829570.429490:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570]  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829570.429492:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829570.429496:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787829570.429504:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000010:3.0:1787829570.429505:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000400:00000001:1.0:1787829570.429506:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829570.429510:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000100:00000001:3.0:1787829570.429510:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429514:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829570.429518:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00100000:3.0:1787829570.429519:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829570.429521:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829570.429528:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829570.429533:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829570.429537:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829570.429545:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829570.429546:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429569:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787829570.429570:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.429572:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429575:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:3.0:1787829570.429576:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3728)
00000100:00000001:1.0:1787829570.429577:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829570.429584:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429589:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829570.429594:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429599:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829570.429601:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429602:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.429603:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429605:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829570.429607:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829570.429609:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829570.429612:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829570.429613:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829570.429616:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829570.429621:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327357312:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829570.429625:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429626:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.429635:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327357312/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429640:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829570.429642:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829570.429644:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829570.429646:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829570.429648:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829570.429649:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829570.429650:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829570.429652:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:1.0:1787829570.429676:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829570.429678:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829570.429679:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:1.0:1787829570.429682:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829570.429682:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429684:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429689:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829570.429695:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.429696:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829570.429698:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829570.429699:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829570.429702:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429711:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829570.429712:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.429714:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829570.429715:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829570.429716:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829570.429719:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357440:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829570.429722:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829570.429724:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829570.429726:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.429728:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829570.429731:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829570.429733:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429737:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829570.429740:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829570.429743:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829570.429747:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327357440, portal 10
00000100:00000001:1.0:1787829570.429749:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829570.429750:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00000040:1.0:1787829570.429758:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429766:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829570.429769:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829570.429771:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829570.429781:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327357440, offset 0
00000400:00000200:1.0:1787829570.429785:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829570.429789:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829570.429792:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829570.429794:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829570.429796:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829570.429798:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829570.429799:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829570.429802:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40564:40564:256:4294967295] 192.168.203.116@tcp LPNI seq info [40564:40564:8:4294967295]
00000400:00000200:1.0:1787829570.429805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829570.429809:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829570.429813:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.429815:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829570.429818:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829570.429824:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787829570.429827:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829570.429840:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787829570.429843:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.429845:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829570.429847:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829570.429850:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829570.429855:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829570.429857:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829570.429859:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829570.429862:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429868:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.429869:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429874:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.429875:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829570.429877:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829570.429880:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829570.429883:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829570.429884:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298390539
00000100:00000001:1.0:1787829570.429886:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298391212
00000100:00000001:1.0:1787829570.429888:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298390539
00000100:00000001:1.0:1787829570.429889:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298391212
00000100:00000001:1.0:1787829570.429890:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829570.429892:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.429894:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829570.429896:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570]  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829570.429902:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829570.429903:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829570.429904:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829570.429905:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829570.429906:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829570.429908:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829570.429910:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829570.429915:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429919:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829570.429920:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429921:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.429922:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829570.429924:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429928:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829570.429931:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429935:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829570.429937:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429938:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.429939:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.429940:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829570.429942:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829570.429944:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829570.429946:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829570.429948:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829570.429950:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829570.429955:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357440:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829570.429958:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829570.429960:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429964:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829570.429965:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.429966:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829570.429967:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829570.429968:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829570.429970:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357568:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829570.429972:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829570.429973:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829570.429974:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.429975:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829570.429978:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829570.429979:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.429981:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829570.429982:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829570.429984:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829570.429986:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327357568, portal 4
00000100:00000001:1.0:1787829570.429988:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829570.429988:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:1.0:1787829570.429992:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.429997:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829570.429999:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829570.430000:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829570.430002:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327357568, offset 0
00000400:00000200:1.0:1787829570.430004:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829570.430007:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829570.430009:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829570.430011:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829570.430013:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829570.430015:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829570.430016:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829570.430018:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40565:40565:256:4294967295] 192.168.203.116@tcp LPNI seq info [40565:40565:8:4294967295]
00000400:00000200:1.0:1787829570.430021:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829570.430025:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829570.430028:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.430030:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829570.430032:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829570.430034:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787829570.430037:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829570.430039:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787829570.430041:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.430043:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829570.430045:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829570.430047:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829570.430049:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829570.430050:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829570.430052:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829570.430054:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430059:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.430061:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430065:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.430066:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829570.430068:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829570.430070:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829570.430072:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829570.430074:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298390546
00000100:00000001:1.0:1787829570.430075:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298391212
00000100:00000001:1.0:1787829570.430077:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298390546
00000100:00000001:1.0:1787829570.430078:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298391212
00000100:00000001:1.0:1787829570.430079:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829570.430080:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.430081:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829570.430083:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570]  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829570.430088:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829570.430089:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829570.430089:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829570.430091:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829570.430092:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829570.430093:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829570.430095:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829570.430103:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430110:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829570.430112:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430114:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.430115:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829570.430119:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430126:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829570.430131:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430139:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829570.430142:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430143:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.430145:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430147:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829570.430148:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829570.430151:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829570.430154:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829570.430155:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829570.430158:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829570.430164:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357568:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829570.430166:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829570.430169:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430176:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829570.430178:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.430180:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829570.430182:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829570.430184:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829570.430187:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357696:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829570.430191:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829570.430192:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829570.430194:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829570.430196:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829570.430200:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829570.430202:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829570.430206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829570.430209:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829570.430213:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829570.430216:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327357696, portal 4
00000100:00000001:1.0:1787829570.430218:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829570.430218:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:1.0:1787829570.430223:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430228:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829570.430229:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829570.430231:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829570.430233:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327357696, offset 0
00000400:00000200:1.0:1787829570.430235:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829570.430238:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829570.430240:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829570.430242:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829570.430244:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829570.430246:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829570.430247:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829570.430250:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40566:40566:256:4294967295] 192.168.203.116@tcp LPNI seq info [40566:40566:8:4294967295]
00000400:00000200:1.0:1787829570.430252:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829570.430257:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829570.430260:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.430262:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829570.430265:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829570.430266:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787829570.430269:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829570.430271:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787829570.430274:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829570.430275:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829570.430277:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829570.430279:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829570.430281:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829570.430282:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829570.430284:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829570.430287:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430292:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.430294:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430298:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.430299:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829570.430301:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829570.430303:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829570.430305:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829570.430306:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298390551
00000100:00000001:1.0:1787829570.430308:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298391212
00000100:00000001:1.0:1787829570.430309:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298390551
00000100:00000001:1.0:1787829570.430310:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298391212
00000100:00000001:1.0:1787829570.430312:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829570.430313:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.430315:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829570.430317:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570]  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829570.430322:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829570.430322:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829570.430323:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829570.430325:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829570.430326:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829570.430327:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829570.430329:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829570.430334:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430342:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829570.430343:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430345:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.430346:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829570.430348:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430352:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829570.430354:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430358:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829570.430359:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430360:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829570.430361:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430362:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829570.430363:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829570.430365:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829570.430367:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829570.430368:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829570.430370:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829570.430375:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357696:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829570.430378:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430379:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.430381:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327357440/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430385:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829570.430386:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829570.430388:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829570.430389:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829570.430391:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829570.430392:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829570.430393:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829570.430394:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:1.0:1787829570.430396:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829570.430397:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829570.430398:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:1.0:1787829570.430400:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829570.430401:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430402:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.430404:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327357568/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430408:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829570.430409:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829570.430410:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829570.430411:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829570.430413:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829570.430414:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829570.430415:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829570.430417:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:1.0:1787829570.430419:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829570.430421:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829570.430422:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829570.430424:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829570.430426:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430440:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829570.430444:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327357696/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829570.430451:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829570.430453:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829570.430455:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829570.430458:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829570.430459:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829570.430460:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829570.430463:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829570.430465:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:1.0:1787829570.430467:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829570.430468:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829570.430470:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:1.0:1787829570.430473:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829570.430474:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430476:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829570.430479:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829570.430480:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829570.430481:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829570.430482:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829570.940815:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829570.940830:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829570.940833:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829570.940839:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829570.940843:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829570.940846:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829570.940849:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829570.940852:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829570.940856:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829571.132707:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298391212, at 4298391232
00000100:00000010:3.0:1787829571.132719:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ae0.
00000100:00000040:2.0:1787829571.132720:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298391212, at 4298391232
00000100:00000010:2.0:1787829571.132728:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f60.
00000020:00000001:2.0:1787829571.260820:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829571.260823:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829571.260826:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829571.964778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829571.964784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829571.964786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829571.964792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829571.964795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829571.964799:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829571.964806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829571.964808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829571.964815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829572.284854:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829572.284861:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829572.284866:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829572.988827:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829572.988836:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829572.988838:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829572.988846:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829572.988850:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829572.988856:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829572.988859:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829572.988862:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829572.988866:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829574.012738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829574.012745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829574.012747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829574.012753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829574.012756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829574.012760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829574.012763:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829574.012766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829574.012770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829574.332911:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829574.332918:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829574.332923:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829575.036782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829575.036790:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829575.036792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829575.036798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829575.036802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829575.036805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829575.036808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829575.036810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829575.036813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829575.356795:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829575.356800:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829575.356804:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829575.549031:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829575.549038:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829575.549044:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829575.549053:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549068:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829575.549071:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.549078:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829575.549082:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829575.549087:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829575.549098:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327357824:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829575.549115:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829575.549121:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829575.549126:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.549131:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829575.549137:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829575.549141:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829575.549150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829575.549154:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829575.549160:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829575.549169:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327357824, portal 25
00000100:00000001:1.0:1787829575.549173:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829575.549176:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829575.549185:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549194:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829575.549199:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829575.549205:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829575.549212:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327357824, offset 0
00000400:00000200:1.0:1787829575.549227:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829575.549239:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.549251:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829575.549261:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829575.549268:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829575.549274:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829575.549278:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829575.549288:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40567:40567:256:4294967295] 192.168.203.116@tcp LPNI seq info [40567:40567:8:4294967295]
00000400:00000200:1.0:1787829575.549296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.549310:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829575.549321:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.549328:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829575.549336:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829575.549341:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787829575.549350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829575.549355:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787829575.549362:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.549365:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829575.549370:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829575.549388:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829575.549409:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829575.549412:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829575.549419:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829575.549428:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549450:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.549457:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549474:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.549478:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829575.549484:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829575.549494:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829575.549500:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298396332
00000100:00000010:1.0:1787829575.549508:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63060.
00000100:00000001:1.0:1787829575.549518:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829575.549526:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298396332
00000100:00000010:1.0:1787829575.549532:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63240.
00000100:00000001:1.0:1787829575.549536:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829575.549548:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829575.549557:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.549561:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829575.549570:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575]  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829575.549584:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829575.549587:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829575.549591:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829575.549596:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829575.549599:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829575.549602:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829575.549607:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829575.549614:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549618:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829575.549620:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549622:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.549623:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829575.549626:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549632:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829575.549635:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549640:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829575.549642:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549643:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.549643:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549645:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829575.549646:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829575.549648:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829575.549651:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829575.549652:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829575.549678:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829575.549685:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327357824:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829575.549689:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549690:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.549693:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327357824/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549697:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829575.549699:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829575.549701:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829575.549703:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829575.549705:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829575.549706:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829575.549707:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829575.549709:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829575.549712:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829575.549714:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829575.549715:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829575.549718:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829575.549718:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549720:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549726:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829575.549727:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.549729:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829575.549730:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829575.549731:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829575.549733:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549738:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829575.549739:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.549741:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829575.549742:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829575.549743:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829575.549745:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357952:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829575.549747:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829575.549748:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829575.549750:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.549751:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829575.549753:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829575.549754:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829575.549757:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829575.549759:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829575.549761:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829575.549763:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327357952, portal 10
00000100:00000001:1.0:1787829575.549764:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829575.549765:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829575.549771:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549775:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829575.549777:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829575.549778:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829575.549780:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327357952, offset 0
00000400:00000200:1.0:1787829575.549783:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829575.549786:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.549788:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829575.549797:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829575.549800:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829575.549802:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829575.549803:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829575.549805:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40568:40568:256:4294967295] 192.168.203.116@tcp LPNI seq info [40568:40568:8:4294967295]
00000400:00000200:1.0:1787829575.549808:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.549813:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829575.549816:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.549818:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829575.549821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829575.549823:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787829575.549826:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829575.549828:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787829575.549830:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.549832:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829575.549834:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829575.549836:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829575.549838:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829575.549839:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829575.549841:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829575.549844:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549853:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.549855:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549859:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.549860:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829575.549863:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829575.549866:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829575.549868:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829575.549869:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298395659
00000100:00000001:1.0:1787829575.549871:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298396332
00000100:00000001:1.0:1787829575.549873:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298395659
00000100:00000001:1.0:1787829575.549874:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298396332
00000100:00000001:1.0:1787829575.549876:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829575.549877:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.549878:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829575.549881:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575]  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829575.549885:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829575.549886:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829575.549887:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829575.549894:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829575.549895:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829575.549897:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829575.549899:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829575.549904:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549908:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829575.549909:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549910:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.549911:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829575.549913:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549917:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829575.549919:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549924:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829575.549926:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549927:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.549928:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.549929:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829575.549930:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829575.549932:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829575.549934:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829575.549936:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829575.549940:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829575.549948:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327357952:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829575.549952:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829575.549956:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.549963:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829575.549965:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.549967:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829575.549969:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829575.549971:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829575.549974:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358080:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829575.549978:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829575.549980:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829575.549981:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.549983:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829575.549987:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829575.549989:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829575.549993:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829575.549996:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829575.550000:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829575.550003:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327358080, portal 4
00000100:00000001:1.0:1787829575.550005:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829575.550006:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829575.550012:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550020:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829575.550022:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829575.550025:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829575.550028:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327358080, offset 0
00000400:00000200:1.0:1787829575.550032:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829575.550036:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.550040:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829575.550043:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829575.550045:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829575.550047:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829575.550048:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829575.550050:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40569:40569:256:4294967295] 192.168.203.116@tcp LPNI seq info [40569:40569:8:4294967295]
00000400:00000200:1.0:1787829575.550053:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.550058:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829575.550061:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.550063:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829575.550066:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829575.550068:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787829575.550071:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829575.550072:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787829575.550075:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.550076:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829575.550078:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829575.550081:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829575.550083:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829575.550085:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829575.550088:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829575.550092:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550100:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.550103:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550110:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.550112:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829575.550115:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829575.550118:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829575.550122:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829575.550124:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298395666
00000100:00000001:1.0:1787829575.550127:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298396332
00000100:00000001:1.0:1787829575.550129:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298395666
00000100:00000001:1.0:1787829575.550132:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298396332
00000100:00000001:1.0:1787829575.550134:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829575.550135:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.550136:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829575.550140:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575]  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829575.550145:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829575.550146:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829575.550147:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829575.550149:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829575.550150:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829575.550152:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829575.550154:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829575.550159:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550168:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829575.550169:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550170:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.550171:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829575.550173:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550178:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829575.550180:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550185:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829575.550186:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550187:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.550188:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550189:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829575.550190:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829575.550193:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829575.550196:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829575.550198:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829575.550202:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829575.550209:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358080:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829575.550213:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829575.550217:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550225:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829575.550227:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.550230:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829575.550231:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829575.550233:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829575.550237:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358208:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829575.550240:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829575.550242:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829575.550243:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829575.550244:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829575.550246:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829575.550248:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829575.550250:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829575.550252:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829575.550254:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829575.550256:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327358208, portal 4
00000100:00000001:1.0:1787829575.550258:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829575.550259:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829575.550263:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550267:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829575.550269:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829575.550270:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829575.550272:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327358208, offset 0
00000400:00000200:1.0:1787829575.550274:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829575.550277:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.550279:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829575.550281:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829575.550283:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829575.550284:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829575.550286:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829575.550288:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40570:40570:256:4294967295] 192.168.203.116@tcp LPNI seq info [40570:40570:8:4294967295]
00000400:00000200:1.0:1787829575.550291:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829575.550295:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829575.550298:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.550300:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829575.550303:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829575.550305:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787829575.550307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829575.550310:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787829575.550312:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829575.550313:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829575.550316:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829575.550318:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829575.550320:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829575.550321:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829575.550323:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829575.550325:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550330:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.550332:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550336:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.550337:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829575.550339:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829575.550341:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829575.550343:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829575.550344:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298395671
00000100:00000001:1.0:1787829575.550346:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298396332
00000100:00000001:1.0:1787829575.550347:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298395671
00000100:00000001:1.0:1787829575.550349:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298396332
00000100:00000001:1.0:1787829575.550350:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829575.550351:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.550353:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829575.550355:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575]  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829575.550360:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829575.550361:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829575.550362:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829575.550364:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829575.550365:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829575.550366:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829575.550368:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829575.550373:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550377:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829575.550378:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550379:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.550380:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829575.550382:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550386:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829575.550389:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550406:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829575.550408:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550409:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829575.550409:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550411:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829575.550412:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829575.550413:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829575.550416:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829575.550417:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829575.550420:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829575.550426:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358208:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829575.550428:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550429:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.550431:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327357952/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550435:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829575.550436:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829575.550438:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829575.550439:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829575.550440:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829575.550441:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829575.550443:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829575.550444:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829575.550446:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829575.550447:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829575.550448:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829575.550450:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829575.550450:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550452:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.550453:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327358080/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550458:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829575.550459:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829575.550460:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829575.550461:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829575.550462:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829575.550462:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829575.550463:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829575.550464:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:1.0:1787829575.550465:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829575.550467:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829575.550468:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829575.550469:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829575.550470:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550471:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829575.550473:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327358208/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829575.550477:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829575.550477:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829575.550478:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829575.550479:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829575.550480:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829575.550481:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829575.550482:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829575.550482:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829575.550484:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829575.550484:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829575.550485:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829575.550486:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829575.550487:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550488:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829575.550491:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829575.550491:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829575.550493:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829575.550494:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829576.060734:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829576.060742:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829576.060745:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829576.060751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829576.060755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829576.060758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829576.060761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829576.060764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829576.060767:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829576.252727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298396332, at 4298396352
00000100:00000040:2.0:1787829576.252729:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298396332, at 4298396352
00000100:00000010:3.0:1787829576.252745:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63240.
00000100:00000010:2.0:1787829576.252748:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63060.
00000100:00000001:2.0:1787829577.084770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829577.084776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829577.084777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829577.084781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829577.084784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829577.084787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829577.084789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829577.084791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829577.084793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829577.404846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829577.404850:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829577.404853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829578.108747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829578.108755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829578.108757:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829578.108762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829578.108765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829578.108769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829578.108772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829578.108775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829578.108778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829578.428917:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829578.428921:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829578.428925:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829579.132771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829579.132778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829579.132781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829579.132786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829579.132790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829579.132793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829579.132795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829579.132797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829579.132800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829580.156774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829580.156780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829580.156781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829580.156785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829580.156788:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829580.156791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829580.156793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829580.156795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829580.156797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829580.476841:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829580.476844:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829580.476847:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787829580.669019:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829580.669025:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829580.669030:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829580.669041:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669064:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829580.669068:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.669077:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829580.669083:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829580.669090:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829580.669099:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327358336:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829580.669115:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829580.669124:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829580.669129:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.669135:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829580.669143:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829580.669148:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829580.669157:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829580.669164:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829580.669176:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829580.669186:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327358336, portal 25
00000100:00000001:1.0:1787829580.669191:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829580.669195:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829580.669207:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669226:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829580.669233:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829580.669239:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829580.669246:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327358336, offset 0
00000400:00000200:1.0:1787829580.669259:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829580.669271:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.669283:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829580.669292:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829580.669301:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829580.669309:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829580.669317:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829580.669327:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40571:40571:256:4294967295] 192.168.203.116@tcp LPNI seq info [40571:40571:8:4294967295]
00000400:00000200:1.0:1787829580.669340:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.669375:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829580.669389:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.669398:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829580.669410:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829580.669419:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829580.669429:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829580.669437:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829580.669447:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.669454:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829580.669462:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829580.669473:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829580.669482:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829580.669487:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829580.669494:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829580.669504:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669525:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.669532:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669551:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.669556:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829580.669565:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829580.669575:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829580.669582:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298401452
00000100:00000010:1.0:1787829580.669590:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63840.
00000100:00000001:1.0:1787829580.669600:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829580.669608:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298401452
00000100:00000010:1.0:1787829580.669614:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e631e0.
00000100:00000001:1.0:1787829580.669621:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829580.669628:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829580.669634:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.669640:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829580.669650:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580]  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829580.669713:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829580.669717:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829580.669722:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829580.669729:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829580.669734:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829580.669740:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829580.669748:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829580.669768:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669791:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829580.669796:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.669802:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.669806:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829580.669817:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669831:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829580.669838:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669846:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829580.669850:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.669851:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.669853:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.669855:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829580.669857:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829580.669860:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829580.669864:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829580.669867:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829580.669872:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829580.669881:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327358336:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829580.669888:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.669891:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.669894:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327358336/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669902:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829580.669905:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829580.669907:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829580.669911:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829580.669913:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829580.669914:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829580.669917:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829580.669920:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:1.0:1787829580.669925:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829580.669927:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829580.669929:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829580.669932:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829580.669933:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.669936:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.669943:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829580.669945:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.669948:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829580.669950:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829580.669951:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829580.669956:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.669963:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829580.669965:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.669967:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829580.669969:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829580.669972:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829580.669975:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358464:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829580.669979:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829580.669981:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829580.669983:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.669985:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829580.669989:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829580.669990:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829580.669995:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829580.669998:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829580.670001:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829580.670005:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327358464, portal 10
00000100:00000001:1.0:1787829580.670007:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829580.670009:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829580.670016:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670023:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829580.670026:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829580.670028:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829580.670031:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327358464, offset 0
00000400:00000200:1.0:1787829580.670036:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829580.670040:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.670044:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829580.670047:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829580.670050:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829580.670052:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829580.670055:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829580.670058:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40572:40572:256:4294967295] 192.168.203.116@tcp LPNI seq info [40572:40572:8:4294967295]
00000400:00000200:1.0:1787829580.670063:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.670071:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829580.670076:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.670079:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829580.670084:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829580.670087:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829580.670092:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829580.670095:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829580.670100:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.670102:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829580.670105:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829580.670109:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829580.670112:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829580.670114:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829580.670117:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829580.670120:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670129:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.670132:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670139:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.670141:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829580.670144:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829580.670148:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829580.670152:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829580.670154:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298400779
00000100:00000001:1.0:1787829580.670157:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298401452
00000100:00000001:1.0:1787829580.670159:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298400779
00000100:00000001:1.0:1787829580.670162:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298401452
00000100:00000001:1.0:1787829580.670164:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829580.670166:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.670168:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829580.670172:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580]  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829580.670180:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829580.670181:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829580.670183:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829580.670186:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829580.670187:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829580.670189:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829580.670194:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829580.670202:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670209:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829580.670211:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670212:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.670214:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829580.670217:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670225:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829580.670230:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670237:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829580.670240:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670242:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.670243:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670245:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829580.670247:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829580.670250:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829580.670254:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829580.670256:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829580.670260:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829580.670269:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358464:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829580.670273:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829580.670277:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670290:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829580.670291:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.670294:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829580.670296:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829580.670298:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829580.670301:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358592:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829580.670304:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829580.670306:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829580.670308:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.670310:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829580.670313:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829580.670315:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829580.670319:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829580.670321:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829580.670325:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829580.670328:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327358592, portal 4
00000100:00000001:1.0:1787829580.670330:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829580.670331:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829580.670338:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670345:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829580.670347:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829580.670350:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829580.670352:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327358592, offset 0
00000400:00000200:1.0:1787829580.670370:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829580.670375:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.670379:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829580.670382:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829580.670385:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829580.670387:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829580.670390:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829580.670394:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40573:40573:256:4294967295] 192.168.203.116@tcp LPNI seq info [40573:40573:8:4294967295]
00000400:00000200:1.0:1787829580.670398:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.670404:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829580.670409:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.670413:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829580.670417:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829580.670420:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829580.670424:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829580.670427:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829580.670431:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.670433:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829580.670437:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829580.670441:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829580.670443:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829580.670445:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829580.670448:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829580.670452:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670461:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.670464:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670471:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.670473:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829580.670476:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829580.670479:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829580.670482:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829580.670485:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298400786
00000100:00000001:1.0:1787829580.670487:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298401452
00000100:00000001:1.0:1787829580.670489:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298400786
00000100:00000001:1.0:1787829580.670492:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298401452
00000100:00000001:1.0:1787829580.670494:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829580.670497:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.670499:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829580.670503:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580]  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829580.670511:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829580.670513:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829580.670514:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829580.670517:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829580.670519:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829580.670521:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829580.670524:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829580.670533:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670540:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829580.670541:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670543:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.670545:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829580.670549:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670557:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829580.670561:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670568:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829580.670571:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670573:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.670574:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670576:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829580.670577:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829580.670580:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829580.670584:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829580.670586:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829580.670590:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829580.670597:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358592:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829580.670601:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829580.670605:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670612:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829580.670614:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.670616:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829580.670618:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829580.670620:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829580.670623:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358720:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829580.670626:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829580.670628:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829580.670629:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829580.670631:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829580.670635:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829580.670637:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829580.670640:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829580.670643:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829580.670646:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829580.670649:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327358720, portal 4
00000100:00000001:1.0:1787829580.670651:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829580.670653:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829580.670676:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670683:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829580.670686:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829580.670689:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829580.670692:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327358720, offset 0
00000400:00000200:1.0:1787829580.670696:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829580.670700:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.670703:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829580.670706:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829580.670709:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829580.670712:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829580.670714:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829580.670718:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40574:40574:256:4294967295] 192.168.203.116@tcp LPNI seq info [40574:40574:8:4294967295]
00000400:00000200:1.0:1787829580.670722:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829580.670728:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829580.670732:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.670736:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829580.670740:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829580.670743:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787829580.670748:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829580.670751:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787829580.670755:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829580.670757:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829580.670760:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829580.670763:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829580.670766:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829580.670768:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829580.670771:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829580.670774:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670783:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.670786:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670793:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.670795:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829580.670799:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829580.670802:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829580.670805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829580.670807:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298400792
00000100:00000001:1.0:1787829580.670809:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298401452
00000100:00000001:1.0:1787829580.670812:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298400792
00000100:00000001:1.0:1787829580.670815:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298401452
00000100:00000001:1.0:1787829580.670816:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829580.670819:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.670821:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829580.670825:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580]  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829580.670833:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829580.670835:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829580.670837:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829580.670839:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829580.670841:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829580.670843:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829580.670846:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829580.670856:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670863:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829580.670865:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670866:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.670868:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829580.670871:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670878:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829580.670882:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670889:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829580.670891:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670893:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829580.670894:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670896:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829580.670898:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829580.670901:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829580.670904:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829580.670906:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829580.670911:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829580.670920:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327358720:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829580.670924:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670926:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.670929:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327358464/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670937:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829580.670938:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829580.670940:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829580.670943:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829580.670944:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829580.670946:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829580.670948:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829580.670950:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829580.670952:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829580.670954:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829580.670955:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829580.670958:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829580.670960:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670962:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.670966:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327358592/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.670973:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829580.670974:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829580.670976:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829580.670978:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829580.670980:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829580.670981:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829580.670982:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829580.670984:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829580.670986:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829580.670987:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829580.670988:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829580.670991:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829580.670992:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.670994:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829580.670997:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327358720/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829580.671004:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829580.671005:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829580.671007:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829580.671009:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829580.671011:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829580.671012:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829580.671014:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829580.671015:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829580.671017:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829580.671019:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829580.671020:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829580.671023:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829580.671024:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.671026:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829580.671029:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829580.671030:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829580.671032:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829580.671034:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829581.180777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829581.180786:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829581.180788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829581.180793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829581.180797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829581.180801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829581.180804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829581.180806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829581.180810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829581.372755:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298401452, at 4298401472
00000100:00000040:3.0:1787829581.372764:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298401452, at 4298401472
00000100:00000010:2.0:1787829581.372773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63840.
00000100:00000010:3.0:1787829581.372781:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e631e0.
00000020:00000001:3.0:1787829581.500873:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829581.500878:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829581.500882:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829582.204766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829582.204774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829582.204777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829582.204782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829582.204786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829582.204790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829582.204795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829582.204798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829582.204801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829583.228749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829583.228754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829583.228756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829583.228760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829583.228763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829583.228765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829583.228767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829583.228769:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829583.228771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829583.548864:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829583.548869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829583.548873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829584.252792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829584.252800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829584.252802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829584.252808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829584.252812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829584.252815:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829584.252818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829584.252821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829584.252824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829584.572841:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829584.572846:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829584.572849:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829585.276758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829585.276766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829585.276769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829585.276774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829585.276777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829585.276780:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829585.276781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829585.276783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829585.276786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829585.788790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829585.788800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829585.788803:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829585.788806:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829585.788811:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829585.788828:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3733
00000400:00000200:1.0:1787829585.788836:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829585.788842:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829585.788846:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829585.788849:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.788853:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.788855:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.788858:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829585.788862:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829585.788864:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829585.788867:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829585.788870:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829585.788873:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829585.788876:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829585.788879:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.788881:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.788884:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.788886:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829585.788889:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829585.788891:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829585.788893:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829585.788895:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829585.788898:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829585.788900:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.788902:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829585.788905:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829585.788913:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829585.788915:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829585.788918:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829585.788919:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.788922:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829585.788924:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829585.788928:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829585.788933:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829585.788938:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829585.788945:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327358848/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829585.788952:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829585.788957:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829585.788962:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327358848/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829585.788984:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829585.788988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829585.788998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829585.789000:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829585.789002:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829585.789006:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829585.789009:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3733
00000400:00000200:1.0:1787829585.789014:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829585.789017:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829585.789021:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829585.789028:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789033:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.789040:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789046:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829585.789052:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829585.789055:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829585.789059:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829585.789064:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829585.789069:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829585.789074:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829585.789082:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789087:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.789091:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789097:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829585.789102:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829585.789109:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789114:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829585.789118:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829585.789127:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829585.789132:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789135:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829585.789139:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829585.789142:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789143:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829585.789156:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829585.789161:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829585.789164:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829585.789166:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789170:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829585.789172:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829585.789175:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829585.789179:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829585.789184:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829585.789191:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327358976/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829585.789202:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829585.789204:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829585.789210:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327358976/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829585.789217:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829585.789220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829585.789225:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829585.789229:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829585.789233:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829585.789237:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829585.789241:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3733
00000400:00000200:1.0:1787829585.789260:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829585.789267:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829585.789270:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829585.789277:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789279:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.789284:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789287:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829585.789291:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829585.789293:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829585.789294:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829585.789297:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829585.789301:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829585.789303:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829585.789305:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789306:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.789311:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789315:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829585.789318:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829585.789338:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789340:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829585.789344:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787829585.789345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:1.0:1787829585.789347:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829585.789351:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829585.789352:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1787829585.789353:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829585.789356:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829585.789359:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789362:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829585.789365:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829585.789368:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829585.789374:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829585.789375:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789376:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829585.789378:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829585.789379:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829585.789386:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829585.789391:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829585.789399:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327359104/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829585.789410:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829585.789412:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829585.789426:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327359104/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829585.789444:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829585.789450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829585.789453:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829585.789457:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829585.789458:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829585.789461:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829585.789466:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3733
00000400:00000200:1.0:1787829585.789474:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829585.789480:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829585.789486:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829585.789492:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789496:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.789502:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789507:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829585.789512:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829585.789515:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829585.789518:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829585.789521:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829585.789526:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829585.789528:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829585.789533:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789537:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829585.789541:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789546:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829585.789549:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829585.789554:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829585.789557:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829585.789561:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829585.789565:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829585.789566:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789568:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829585.789569:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829585.789571:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789573:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829585.789579:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829585.789586:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829585.789592:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829585.789596:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829585.789600:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829585.789604:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829585.789608:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829585.789624:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829585.789631:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829585.789641:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327359232/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829585.789698:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829585.789704:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829585.789717:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327359232/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829585.789746:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829585.789752:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3743)
00000800:00000200:0.1:1787829585.790007:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829585.790010:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829585.790349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829585.790365:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829585.790647:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829585.790650:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787829586.300725:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829586.300731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829586.300732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829586.300736:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829586.300739:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829586.300742:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829586.300744:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829586.300747:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829586.300750:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829586.492778:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298406572, at 4298406592
00000100:00000040:3.0:1787829586.492778:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298406572, at 4298406592
00000100:00000010:3.0:1787829586.492796:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000100:00000010:2.0:1787829586.492799:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000020:00000001:2.0:1787829586.620826:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829586.620830:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829586.620834:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829587.324808:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829587.324818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829587.324819:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829587.324826:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829587.324829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829587.324833:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829587.324835:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829587.324838:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829587.324841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829587.644892:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829587.644895:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829587.644898:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829588.348766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829588.348772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829588.348774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829588.348778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829588.348781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829588.348784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829588.348786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829588.348788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829588.348790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829589.372741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829589.372749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829589.372752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829589.372757:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829589.372761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829589.372765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829589.372767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829589.372771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829589.372775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829589.692879:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829589.692883:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829589.692887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829590.396776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829590.396782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829590.396783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829590.396788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829590.396790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829590.396793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829590.396795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829590.396797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829590.396799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829590.716799:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829590.716803:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829590.716807:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829590.908774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829590.908782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829590.908785:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829590.908788:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829590.908792:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829590.908799:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3738
00000400:00000200:2.0:1787829590.908806:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829590.908814:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829590.908817:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829590.908820:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.908823:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.908826:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.908828:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829590.908832:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829590.908834:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829590.908837:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829590.908841:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829590.908843:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829590.908847:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829590.908850:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.908853:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.908855:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.908857:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829590.908860:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829590.908874:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829590.908877:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829590.908880:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829590.908884:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829590.908886:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.908889:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829590.908892:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829590.908901:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787829590.908905:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829590.908909:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829590.908911:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.908915:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829590.908917:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829590.908922:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829590.908929:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829590.908934:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829590.908942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327359360/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829590.908949:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829590.908953:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829590.908956:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327359360/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829590.908974:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829590.908976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829590.908979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829590.908980:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829590.908982:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829590.908984:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829590.908986:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3738
00000400:00000200:2.0:1787829590.908989:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829590.908991:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829590.908992:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829590.908995:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.908997:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.908999:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909001:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829590.909003:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829590.909005:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829590.909007:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829590.909010:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829590.909012:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829590.909014:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829590.909018:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909023:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.909026:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909032:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829590.909037:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829590.909043:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909049:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829590.909054:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829590.909060:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829590.909063:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909067:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829590.909072:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829590.909085:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909088:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829590.909096:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787829590.909100:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829590.909103:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829590.909106:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909108:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829590.909110:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829590.909114:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829590.909118:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829590.909124:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829590.909136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327359488/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829590.909153:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829590.909159:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829590.909167:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327359488/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829590.909185:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829590.909189:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829590.909192:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829590.909195:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829590.909198:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829590.909202:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829590.909207:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3738
00000400:00000200:2.0:1787829590.909216:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829590.909222:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829590.909226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829590.909229:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909233:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.909235:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909236:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829590.909238:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829590.909238:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829590.909240:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829590.909241:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829590.909243:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829590.909244:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829590.909248:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909249:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.909252:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909255:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829590.909257:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829590.909262:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909266:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829590.909279:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829590.909283:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829590.909298:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909300:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829590.909304:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829590.909308:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909312:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829590.909316:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787829590.909319:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829590.909323:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829590.909326:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909329:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829590.909332:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829590.909335:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829590.909339:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829590.909345:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829590.909350:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327359616/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829590.909367:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829590.909372:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829590.909376:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327359616/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829590.909391:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829590.909404:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829590.909413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829590.909417:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829590.909421:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829590.909427:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829590.909433:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3738
00000800:00000200:0.1:1787829590.909442:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:2.0:1787829590.909443:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.1:1787829590.909451:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000400:00000200:2.0:1787829590.909452:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829590.909456:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829590.909461:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909466:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.909471:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909476:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829590.909480:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829590.909483:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829590.909486:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829590.909490:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829590.909496:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829590.909501:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829590.909506:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909512:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829590.909516:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909521:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829590.909526:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829590.909533:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829590.909541:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829590.909546:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829590.909551:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829590.909556:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909560:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829590.909564:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829590.909572:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909577:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829590.909585:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787829590.909590:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829590.909596:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829590.909599:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829590.909603:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829590.909607:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829590.909610:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829590.909616:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829590.909622:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829590.909633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327359744/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829590.909642:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829590.909646:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829590.909651:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327359744/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829590.909688:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829590.909696:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3748)
00000800:00000200:0.1:1787829590.910040:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829590.910043:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829590.910377:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829590.910380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829590.910708:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829590.910711:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1787829591.420747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829591.420752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829591.420754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829591.420759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829591.420761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829591.420764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829591.420765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829591.420767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829591.420769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829591.612711:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298411692, at 4298411712
00000100:00000040:3.0:1787829591.612718:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298411692, at 4298411712
00000100:00000010:2.0:1787829591.612725:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00000010:3.0:1787829591.612729:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000001:2.0:1787829592.444793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829592.444801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829592.444803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829592.444809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829592.444813:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829592.444817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829592.444820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829592.444822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829592.444825:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829592.764802:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829592.764806:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829592.764809:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829593.468751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829593.468757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829593.468758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829593.468762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829593.468765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829593.468768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829593.468769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829593.468771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829593.468773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829593.788843:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829593.788846:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829593.788850:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829594.492795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829594.492802:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829594.492805:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829594.492810:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829594.492814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829594.492818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829594.492820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829594.492824:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829594.492827:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829595.516762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829595.516769:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829595.516770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829595.516774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829595.516777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829595.516784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829595.516785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829595.516787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829595.516789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829595.836846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829595.836851:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829595.836855:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829596.028798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829596.028809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829596.028812:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829596.028816:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829596.028821:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829596.028828:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3743
00000400:00000200:3.0:1787829596.028838:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829596.028845:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829596.028848:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829596.028852:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.028856:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.028859:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.028862:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829596.028866:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829596.028868:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829596.028871:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829596.028874:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829596.028877:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829596.028880:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829596.028883:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.028885:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.028888:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.028890:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829596.028893:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829596.028897:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829596.028899:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829596.028903:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829596.028907:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829596.028910:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.028913:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829596.028915:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829596.028932:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829596.028937:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829596.028941:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829596.028943:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.028946:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829596.028949:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829596.028954:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829596.028961:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829596.028968:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829596.028977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327359872/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829596.028988:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829596.029000:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829596.029006:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327359872/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829596.029029:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829596.029033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829596.029036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829596.029038:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829596.029039:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829596.029043:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829596.029046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3743
00000400:00000200:3.0:1787829596.029051:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829596.029055:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829596.029058:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829596.029061:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029065:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.029069:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029074:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829596.029079:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829596.029081:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829596.029084:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829596.029088:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829596.029093:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829596.029098:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829596.029106:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029111:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.029116:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029119:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829596.029123:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829596.029129:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029135:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829596.029139:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829596.029146:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829596.029151:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029155:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829596.029163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829596.029168:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029173:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829596.029181:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829596.029185:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829596.029190:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829596.029193:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029196:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829596.029198:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829596.029200:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829596.029209:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829596.029215:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829596.029224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327360000/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829596.029241:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829596.029245:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829596.029267:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327360000/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829596.029282:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829596.029287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829596.029293:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829596.029295:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829596.029298:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829596.029305:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829596.029309:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3743
00000400:00000200:3.0:1787829596.029317:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829596.029324:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829596.029328:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829596.029334:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029338:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.029343:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029348:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829596.029352:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829596.029356:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829596.029360:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829596.029363:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829596.029369:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829596.029373:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829596.029388:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029393:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.029400:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029405:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829596.029409:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829596.029415:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029421:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829596.029424:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829596.029428:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829596.029432:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029436:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829596.029439:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829596.029444:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029448:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829596.029454:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000800:00000200:0.1:1787829596.029456:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787829596.029458:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829596.029463:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.1:1787829596.029463:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
02000000:00000001:3.0:1787829596.029472:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029476:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829596.029480:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829596.029483:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829596.029487:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829596.029493:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829596.029502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327360128/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829596.029517:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829596.029521:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829596.029529:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327360128/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829596.029551:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829596.029556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829596.029561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829596.029565:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829596.029567:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829596.029573:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829596.029584:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3743
00000400:00000200:3.0:1787829596.029595:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829596.029599:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829596.029601:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829596.029605:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029608:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.029613:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029618:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829596.029622:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829596.029626:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829596.029627:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829596.029630:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829596.029635:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829596.029638:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829596.029641:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029646:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829596.029651:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029691:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829596.029697:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829596.029704:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829596.029710:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829596.029713:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829596.029717:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829596.029724:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029728:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829596.029731:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829596.029736:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029740:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829596.029746:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829596.029751:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829596.029756:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829596.029760:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829596.029763:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829596.029767:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829596.029770:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829596.029777:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829596.029784:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829596.029793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327360256/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829596.029809:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829596.029813:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829596.029821:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327360256/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829596.029839:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829596.029848:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3753)
00000800:00000200:0.1:1787829596.030099:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829596.030102:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829596.030441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829596.030444:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829596.030721:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829596.030723:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1787829596.540756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829596.540762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829596.540764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829596.540770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829596.540774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829596.540778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829596.540781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829596.540784:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829596.540787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829596.732780:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298416812, at 4298416832
00000100:00000040:3.0:1787829596.732780:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298416812, at 4298416832
00000100:00000010:2.0:1787829596.732797:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000100:00000010:3.0:1787829596.732800:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000020:00000001:3.0:1787829596.860852:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829596.860856:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829596.860861:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829597.564737:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829597.564745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829597.564747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829597.564754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829597.564758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829597.564761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829597.564764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829597.564766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829597.564769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829598.588755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829598.588761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829598.588763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829598.588767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829598.588769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829598.588772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829598.588775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829598.588778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829598.588782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829598.908856:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829598.908860:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829598.908864:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829599.612800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829599.612807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829599.612810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829599.612816:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829599.612820:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829599.612823:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829599.612825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829599.612829:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829599.612831:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829599.932891:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829599.932896:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829599.932904:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829600.636782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829600.636790:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829600.636793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829600.636805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829600.636808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829600.636811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829600.636813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829600.636815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829600.636817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829601.148797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829601.148808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829601.148811:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829601.148815:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829601.148821:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829601.148830:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3748
00000400:00000200:3.0:1787829601.148841:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829601.148849:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829601.148854:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829601.148860:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.148863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.148867:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.148869:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829601.148873:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829601.148875:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829601.148878:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829601.148881:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829601.148883:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829601.148885:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829601.148887:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.148888:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.148889:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.148890:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829601.148892:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829601.148895:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829601.148897:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829601.148900:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829601.148905:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829601.148907:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.148910:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829601.148913:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829601.148925:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829601.148928:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829601.148933:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829601.148935:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.148939:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829601.148943:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829601.148948:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829601.148957:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829601.148966:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829601.148977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327360384/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829601.148988:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829601.148994:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829601.149000:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327360384/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829601.149023:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829601.149027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829601.149030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829601.149031:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829601.149033:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829601.149037:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829601.149041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3748
00000400:00000200:3.0:1787829601.149047:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829601.149050:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829601.149053:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829601.149056:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149068:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.149072:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149081:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829601.149084:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829601.149086:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829601.149090:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829601.149095:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829601.149104:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829601.149115:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829601.149129:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149139:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.149153:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149169:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829601.149178:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829601.149188:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149201:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829601.149209:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829601.149230:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829601.149241:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.149244:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829601.149250:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829601.149259:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.149271:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829601.149286:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829601.149291:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829601.149302:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829601.149307:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.149313:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829601.149316:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829601.149319:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829601.149325:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829601.149333:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829601.149343:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327360512/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829601.149401:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829601.149406:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829601.149427:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327360512/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829601.149427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829601.149438:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:3.0:1787829601.149466:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829601.149476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829601.149487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829601.149493:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829601.149498:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829601.149507:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829601.149516:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3748
00000400:00000200:3.0:1787829601.149524:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829601.149534:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829601.149538:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829601.149544:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149548:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.149553:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149558:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829601.149566:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829601.149568:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829601.149573:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829601.149579:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829601.149586:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829601.149592:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829601.149605:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149617:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.149622:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149627:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829601.149632:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829601.149639:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149648:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829601.149687:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829601.149692:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829601.149695:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.149696:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829601.149698:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829601.149701:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.149710:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829601.149723:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829601.149729:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829601.149737:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829601.149742:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.149747:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829601.149759:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829601.149769:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829601.149780:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829601.149800:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829601.149815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327360640/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829601.149880:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829601.149886:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829601.149892:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327360640/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829601.149906:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829601.149915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829601.149929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829601.149931:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829601.149932:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829601.149934:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829601.149939:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3748
00000400:00000200:3.0:1787829601.149951:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829601.149964:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829601.149970:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829601.149980:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.149986:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.150002:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.150015:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829601.150021:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829601.150026:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829601.150030:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829601.150034:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829601.150041:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829601.150051:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829601.150068:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.150076:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829601.150088:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829601.150096:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829601.150106:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829601.150117:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829601.150126:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829601.150143:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829601.150150:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829601.150158:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.150164:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829601.150171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829601.150176:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.150185:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829601.150195:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829601.150205:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829601.150211:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829601.150225:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829601.150228:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829601.150233:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829601.150236:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829601.150248:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829601.150258:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829601.150274:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327360768/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829601.150285:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829601.150291:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829601.150303:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327360768/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829601.150331:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829601.150343:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3758)
00000800:00000200:0.1:1787829601.150401:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829601.150403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829601.150620:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829601.150634:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829601.150921:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829601.150923:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1787829601.660815:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829601.660821:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829601.660822:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829601.660826:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829601.660829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829601.660832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829601.660834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829601.660836:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829601.660838:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829601.852782:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298421932, at 4298421952
00000100:00000040:3.0:1787829601.852782:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298421932, at 4298421952
00000100:00000010:3.0:1787829601.852804:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000100:00000010:2.0:1787829601.852808:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000020:00000001:2.0:1787829601.980875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829601.980880:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829601.980886:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829602.684765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829602.684772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829602.684774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829602.684779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829602.684783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829602.684787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829602.684790:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829602.684793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829602.684796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829603.004814:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829603.004818:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829603.004823:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829605.052862:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829605.052866:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829605.052869:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829606.076828:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829606.076832:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829606.076834:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829606.268739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829606.268749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829606.268751:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829606.268754:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829606.268758:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829606.268764:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3753
00000400:00000200:3.0:1787829606.268772:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829606.268778:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829606.268787:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829606.268789:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268792:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.268793:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268795:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829606.268798:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829606.268799:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829606.268801:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829606.268804:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829606.268806:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829606.268809:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829606.268812:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268814:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.268816:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268818:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829606.268822:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829606.268826:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268830:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829606.268832:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829606.268836:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829606.268839:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.268841:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829606.268844:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829606.268853:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829606.268857:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829606.268861:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829606.268863:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.268866:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829606.268868:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829606.268872:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829606.268878:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829606.268883:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829606.268891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327360896/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829606.268899:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829606.268902:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829606.268906:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327360896/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829606.268927:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829606.268930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829606.268934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829606.268935:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829606.268937:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829606.268942:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829606.268945:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3753
00000400:00000200:3.0:1787829606.268951:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829606.268956:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829606.268959:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829606.268963:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268966:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.268972:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.268976:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829606.268980:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829606.268981:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829606.268983:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829606.268987:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829606.268991:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829606.268995:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829606.269002:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269007:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.269011:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269014:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829606.269017:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829606.269023:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269027:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829606.269029:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829606.269035:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829606.269039:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269042:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829606.269048:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829606.269051:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269053:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829606.269060:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829606.269064:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829606.269069:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829606.269072:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269077:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829606.269078:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829606.269081:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829606.269090:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829606.269093:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829606.269103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327361024/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829606.269111:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829606.269115:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829606.269125:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327361024/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829606.269146:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829606.269152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829606.269160:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829606.269163:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829606.269167:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829606.269172:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829606.269196:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3753
00000400:00000200:3.0:1787829606.269203:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829606.269210:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829606.269214:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829606.269222:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269226:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.269230:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269234:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829606.269239:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829606.269242:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829606.269244:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829606.269246:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829606.269251:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829606.269257:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829606.269264:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269267:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.269271:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269276:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829606.269278:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829606.269282:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269285:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829606.269286:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829606.269291:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829606.269295:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269296:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829606.269300:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829606.269305:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829606.269305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829606.269309:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000100:00000001:3.0:1787829606.269312:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829606.269317:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829606.269323:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829606.269327:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829606.269330:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269334:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829606.269337:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829606.269340:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829606.269344:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829606.269350:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829606.269357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327361152/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829606.269378:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829606.269383:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829606.269393:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327361152/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829606.269413:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829606.269419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829606.269426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829606.269429:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829606.269432:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829606.269438:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829606.269441:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3753
00000400:00000200:3.0:1787829606.269450:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829606.269456:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829606.269459:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829606.269465:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269468:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.269474:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269476:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829606.269482:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829606.269484:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829606.269487:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829606.269491:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829606.269494:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829606.269498:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829606.269502:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269506:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829606.269511:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269516:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829606.269520:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829606.269525:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829606.269530:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829606.269534:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829606.269539:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829606.269544:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269548:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829606.269551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829606.269557:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269559:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829606.269564:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829606.269569:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829606.269574:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829606.269577:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829606.269581:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829606.269583:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829606.269586:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829606.269591:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829606.269596:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829606.269606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327361280/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829606.269623:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829606.269628:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829606.269633:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327361280/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829606.269651:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829606.269682:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3764)
00000800:00000200:0.1:1787829606.269859:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829606.269861:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787829606.270074:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829606.270076:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787829606.270276:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829606.270277:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 11 secs later)
00000100:00000040:2.0:1787829606.972729:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298427052, at 4298427072
00000100:00000040:3.0:1787829606.972736:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298427052, at 4298427072
00000100:00000010:2.0:1787829606.972742:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00000100:00000010:3.0:1787829606.972744:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000020:00000001:2.0:1787829608.124798:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829608.124801:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829608.124804:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:3.0:1787829609.148917:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829609.148924:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829609.148931:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1787829611.196916:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829611.196923:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829611.196929:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829611.388789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829611.388804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829611.388807:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829611.388813:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829611.388818:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829611.388828:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3759
00000400:00000200:3.0:1787829611.388840:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829611.388849:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829611.388855:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829611.388860:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.388873:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.388877:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.388880:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829611.388886:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829611.388889:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829611.388892:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829611.388896:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829611.388899:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829611.388902:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829611.388906:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.388908:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.388911:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.388918:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829611.388923:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829611.388927:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829611.388930:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829611.388934:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829611.388939:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829611.388943:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.388946:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829611.388949:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829611.388961:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829611.388966:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829611.388971:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829611.388973:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.388976:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829611.388981:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829611.388986:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829611.388996:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829611.389005:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829611.389018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327361408/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829611.389031:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829611.389036:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829611.389043:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327361408/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829611.389072:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829611.389077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829611.389080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829611.389082:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829611.389084:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829611.389087:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829611.389091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3759
00000400:00000200:3.0:1787829611.389096:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829611.389100:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829611.389103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829611.389108:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389113:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.389119:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389124:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829611.389130:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829611.389133:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829611.389138:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829611.389156:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829611.389163:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829611.389170:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829611.389179:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389187:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.389195:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389201:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829611.389206:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829611.389211:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389218:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829611.389221:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829611.389230:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829611.389235:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389240:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829611.389246:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829611.389254:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389260:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829611.389269:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829611.389275:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829611.389283:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829611.389287:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389291:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829611.389295:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829611.389299:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829611.389306:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829611.389313:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829611.389323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829611.389336:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829611.389341:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829611.389348:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829611.389365:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829611.389371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829611.389378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829611.389381:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829611.389384:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829611.389391:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829611.389397:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3759
00000400:00000200:3.0:1787829611.389405:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829611.389410:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829611.389413:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829611.389419:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389424:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.389430:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389434:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829611.389438:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829611.389442:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:0.1:1787829611.389447:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:3.0:1787829611.389448:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829611.389453:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:0.1:1787829611.389454:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000020:00000001:3.0:1787829611.389458:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829611.389465:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829611.389474:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389478:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.389481:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389485:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829611.389490:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829611.389498:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389504:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829611.389508:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829611.389514:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829611.389520:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389526:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829611.389532:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829611.389549:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389554:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829611.389563:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829611.389570:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829611.389578:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829611.389582:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389586:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829611.389589:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829611.389594:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829611.389602:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829611.389606:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829611.389616:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829611.389634:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829611.389638:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829611.389648:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829611.389695:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829611.389700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829611.389708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829611.389713:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829611.389716:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829611.389721:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829611.389727:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3759
00000400:00000200:3.0:1787829611.389737:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829611.389744:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829611.389750:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829611.389756:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389762:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.389768:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389775:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829611.389781:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829611.389786:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829611.389790:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829611.389794:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829611.389800:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829611.389806:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829611.389815:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389821:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829611.389827:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389833:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829611.389841:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829611.389850:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829611.389858:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829611.389862:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829611.389867:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829611.389873:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389878:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829611.389882:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829611.389888:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389891:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829611.389898:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829611.389903:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829611.389910:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829611.389913:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829611.389916:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829611.389920:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829611.389925:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829611.389933:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:3.0:1787829611.389940:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829611.389953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829611.389983:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829611.389989:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829611.389998:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829611.390025:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829611.390033:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3769)
00000100:00000040:0.0:1787829611.390063:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829611.390065:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829611.390067:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327361536, offset 0
00000400:00000200:0.0:1787829611.390069:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829611.390073:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829611.390075:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829611.390078:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829611.390080:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829611.390082:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829611.390084:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829611.390086:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40596:40596:256:4294967295] 192.168.203.116@tcp LPNI seq info [40596:40596:8:4294967295]
00000400:00000200:0.0:1787829611.390089:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829611.390094:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829611.390097:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829611.390099:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829611.390102:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829611.390104:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787829611.390108:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829611.390110:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1787829611.390112:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829611.390114:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829611.390116:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829611.390119:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829611.390121:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829611.390122:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829611.390124:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829611.390127:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390132:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829611.390134:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390139:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390140:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829611.390157:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829611.390160:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829611.390163:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829611.390164:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298431499
00000100:00000001:0.0:1787829611.390167:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298432172
00000100:00000001:0.0:1787829611.390168:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298431499
00000100:00000001:0.0:1787829611.390170:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298432172
00000100:00000001:0.0:1787829611.390172:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829611.390173:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390174:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829611.390177:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611]  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829611.390182:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829611.390183:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829611.390184:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829611.390186:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829611.390187:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829611.390189:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829611.390191:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829611.390197:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390201:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829611.390202:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390203:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829611.390204:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829611.390206:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390210:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829611.390212:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390218:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829611.390220:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390221:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829611.390222:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390223:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829611.390224:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829611.390226:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829611.390228:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829611.390229:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829611.390231:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829611.390236:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327361536:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829611.390238:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829611.390240:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390245:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829611.390246:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829611.390247:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829611.390249:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829611.390250:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829611.390252:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327361664:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829611.390254:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829611.390255:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829611.390256:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829611.390257:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829611.390260:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829611.390261:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829611.390264:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829611.390266:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829611.390268:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000200:0.0:1787829611.390270:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327361664, portal 4
00000100:00000001:0.0:1787829611.390272:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829611.390273:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:0.0:1787829611.390277:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390281:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829611.390283:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829611.390284:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829611.390286:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327361664, offset 0
00000400:00000200:0.0:1787829611.390289:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829611.390291:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829611.390294:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829611.390296:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829611.390298:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829611.390300:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829611.390302:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829611.390304:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40597:40597:256:4294967295] 192.168.203.116@tcp LPNI seq info [40597:40597:8:4294967295]
00000400:00000200:0.0:1787829611.390307:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829611.390311:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829611.390316:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829611.390319:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829611.390323:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829611.390326:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787829611.390332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829611.390334:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1787829611.390339:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829611.390341:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829611.390344:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829611.390348:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829611.390351:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829611.390353:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829611.390356:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829611.390361:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390367:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829611.390369:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390374:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390375:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829611.390378:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829611.390380:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829611.390382:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829611.390384:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298431506
00000100:00000001:0.0:1787829611.390386:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298432172
00000100:00000001:0.0:1787829611.390387:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298431506
00000100:00000001:0.0:1787829611.390389:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298432172
00000100:00000001:0.0:1787829611.390390:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829611.390392:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390393:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829611.390395:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611]  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829611.390400:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829611.390401:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829611.390402:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829611.390403:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829611.390405:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829611.390406:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829611.390408:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829611.390413:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390421:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829611.390423:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390424:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829611.390425:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829611.390427:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390431:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829611.390434:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390438:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829611.390439:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390440:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829611.390441:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390442:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829611.390443:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829611.390444:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829611.390446:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829611.390447:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829611.390451:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829611.390459:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327361664:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829611.390462:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390464:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829611.390467:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327361536/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390475:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829611.390477:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829611.390479:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829611.390482:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829611.390484:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829611.390485:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829611.390488:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829611.390490:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829611.390493:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829611.390494:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829611.390496:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:0.0:1787829611.390499:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829611.390501:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390503:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829611.390506:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327361664/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390512:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829611.390513:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829611.390514:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829611.390516:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829611.390517:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829611.390517:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829611.390518:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829611.390519:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829611.390521:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829611.390522:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829611.390523:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829611.390524:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829611.390525:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390527:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390529:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829611.390531:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390532:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829611.390533:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829611.390534:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829611.390537:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390541:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829611.390543:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829611.390544:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829611.390546:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829611.390547:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829611.390549:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327361792:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829611.390551:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829611.390553:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829611.390554:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829611.390556:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829611.390558:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829611.390559:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829611.390562:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829611.390564:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829611.390566:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000200:0.0:1787829611.390568:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327361792, portal 4
00000100:00000001:0.0:1787829611.390569:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829611.390570:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:0.0:1787829611.390575:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390579:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829611.390581:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829611.390582:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829611.390584:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327361792, offset 0
00000400:00000200:0.0:1787829611.390586:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829611.390589:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829611.390591:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829611.390593:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829611.390595:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829611.390597:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829611.390598:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829611.390601:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40598:40598:256:4294967295] 192.168.203.116@tcp LPNI seq info [40598:40598:8:4294967295]
00000400:00000200:0.0:1787829611.390603:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829611.390608:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829611.390612:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829611.390614:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829611.390617:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829611.390619:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787829611.390622:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829611.390624:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1787829611.390626:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829611.390628:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829611.390630:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829611.390632:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829611.390634:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829611.390635:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829611.390637:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829611.390639:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390645:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829611.390647:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390651:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390652:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829611.390654:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829611.390672:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829611.390675:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829611.390676:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298431512
00000100:00000001:0.0:1787829611.390678:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298432172
00000100:00000001:0.0:1787829611.390680:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298431512
00000100:00000001:0.0:1787829611.390681:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298432172
00000100:00000001:0.0:1787829611.390682:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829611.390684:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390685:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829611.390687:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611]  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829611.390694:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829611.390695:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829611.390696:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829611.390698:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829611.390698:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829611.390700:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829611.390702:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829611.390712:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390718:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829611.390720:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390722:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829611.390723:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829611.390727:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390735:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829611.390740:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390746:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829611.390748:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390750:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829611.390751:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390752:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829611.390753:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829611.390755:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829611.390757:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829611.390758:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829611.390762:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829611.390767:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327361792:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829611.390770:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390771:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829611.390773:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327361792/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829611.390777:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829611.390777:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829611.390779:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829611.390780:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829611.390781:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829611.390782:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829611.390784:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829611.390785:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829611.390786:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829611.390787:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829611.390788:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:0.0:1787829611.390791:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829611.390792:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390793:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.390796:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829611.390797:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.390798:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829611.390799:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829611.900781:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829611.900789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829611.900792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829611.900797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829611.900801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829611.900804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.900807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829611.900809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829611.900812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829612.092760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298432172, at 4298432192
00000100:00000040:3.0:1787829612.092760:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298432172, at 4298432192
00000100:00000010:2.0:1787829612.092773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000100:00000010:3.0:1787829612.092784:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00010000:00000001:0.0:1787829612.220747:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829612.220754:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829612.220759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829612.220761:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829612.220764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829612.220765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829612.220811:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829612.220818:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829612.220823:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829612.924736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829612.924743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829612.924746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829612.924752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829612.924755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829612.924758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829612.924760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829612.924762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829612.924764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829613.244755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829613.244760:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829613.244765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829613.244767:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829613.244770:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829613.244772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829613.244788:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829613.244790:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829613.244793:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829613.948804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829613.948812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829613.948814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829613.948821:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829613.948826:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829613.948831:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829613.948834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829613.948838:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829613.948841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829614.268798:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829614.268801:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829614.268803:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268810:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829614.268812:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268814:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268816:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268818:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829614.268820:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829614.268820:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829614.268822:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268823:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829614.268824:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268825:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268826:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829614.268828:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829614.268829:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829614.268884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829614.268889:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829614.268893:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829614.972765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829614.972777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829614.972779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829614.972787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829614.972792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829614.972797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829614.972800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829614.972804:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829614.972808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829615.292810:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829615.292819:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829615.292827:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829615.292829:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829615.292833:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829615.292835:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829615.292891:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829615.292896:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829615.292900:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829615.996800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829615.996811:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829615.996814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829615.996820:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829615.996824:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829615.996830:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829615.996834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829615.996837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829615.996841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829616.316775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829616.316783:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829616.316788:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829616.316789:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829616.316792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829616.316793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829616.316812:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829616.316814:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829616.316817:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829616.508807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829616.508818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829616.508821:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829616.508825:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829616.508830:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829616.508838:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3764
00000400:00000200:3.0:1787829616.508848:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829616.508855:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829616.508858:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829616.508861:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.508863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829616.508865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.508867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829616.508871:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829616.508872:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829616.508875:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829616.508878:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829616.508880:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829616.508882:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829616.508885:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.508887:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829616.508888:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.508890:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829616.508893:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829616.508895:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829616.508897:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829616.508899:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829616.508903:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829616.508906:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.508908:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829616.508911:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829616.508925:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829616.508930:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829616.508935:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829616.508937:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.508941:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829616.508944:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829616.508950:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829616.508961:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829616.508969:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829616.508980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829616.508989:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829616.508994:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829616.508997:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829616.509018:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829616.509021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829616.509024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829616.509025:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829616.509027:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829616.509030:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829616.509032:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3764
00000400:00000200:3.0:1787829616.509035:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829616.509037:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829616.509039:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829616.509042:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509044:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829616.509046:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509048:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829616.509050:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829616.509051:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829616.509052:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829616.509054:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829616.509056:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829616.509057:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829616.509059:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509060:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829616.509063:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787829616.509063:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509065:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829616.509069:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829616.509071:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829616.509072:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829616.509074:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509077:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1787829616.509077:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829616.509083:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829616.509087:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829616.509089:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829616.509089:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829616.509093:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829616.509099:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.509103:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509122:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000010:3.0:1787829616.509122:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829616.509126:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:0.0:1787829616.509127:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829616.509129:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829616.509132:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.509134:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829616.509136:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787829616.509136:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829616.509138:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.509142:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1787829616.509145:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829616.509147:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829616.509154:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327361920:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829616.509155:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509162:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:3.0:1787829616.509164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829616.509169:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787829616.509172:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829616.509174:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.509175:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.509181:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:3.0:1787829616.509182:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829616.509190:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
00000100:00000001:3.0:1787829616.509191:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.509196:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829616.509196:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829616.509202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829616.509204:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829616.509205:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829616.509209:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829616.509209:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829616.509214:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3764
00000400:00000010:0.0:1787829616.509217:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000200:3.0:1787829616.509217:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829616.509223:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829616.509225:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829616.509227:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509229:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829616.509230:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000040:3.0:1787829616.509231:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509235:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829616.509236:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829616.509238:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829616.509240:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:0.0:1787829616.509241:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327361920, portal 25
00000020:00000040:3.0:1787829616.509242:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829616.509246:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:0.0:1787829616.509248:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:3.0:1787829616.509249:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:0.0:1787829616.509252:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:3.0:1787829616.509253:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509259:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829616.509260:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509262:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829616.509266:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000040:0.0:1787829616.509267:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829616.509269:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509274:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829616.509278:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829616.509282:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829616.509286:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509287:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787829616.509288:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829616.509291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829616.509295:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829616.509296:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509298:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829616.509301:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000010:3.0:1787829616.509303:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829616.509306:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000200:0.0:1787829616.509308:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327361920, offset 0
02000000:00000001:3.0:1787829616.509308:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829616.509310:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.509314:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829616.509315:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829616.509316:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829616.509320:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000400:00000200:0.0:1787829616.509323:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509323:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829616.509330:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829616.509335:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829616.509338:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.509339:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829616.509343:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829616.509350:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.509352:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:3.0:1787829616.509353:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829616.509360:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829616.509361:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829616.509363:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829616.509365:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787829616.509366:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829616.509371:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3764
00000400:00000200:3.0:1787829616.509374:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.509376:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787829616.509379:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829616.509382:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829616.509385:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1787829616.509388:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509390:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829616.509393:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:3.0:1787829616.509394:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509398:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829616.509400:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829616.509401:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829616.509404:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829616.509405:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40599:40599:256:4294967295] 192.168.203.116@tcp LPNI seq info [40599:40599:8:4294967295]
00000020:00000040:3.0:1787829616.509406:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829616.509410:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829616.509415:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829616.509419:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829616.509420:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509424:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829616.509426:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509427:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829616.509429:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829616.509432:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829616.509433:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829616.509437:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829616.509440:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829616.509444:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.509445:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787829616.509446:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829616.509449:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829616.509453:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.509454:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829616.509457:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000800:00000200:0.0:1787829616.509458:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000040:3.0:1787829616.509459:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829616.509464:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829616.509465:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829616.509466:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829616.509467:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:0.0:1787829616.509468:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
02000000:00000001:3.0:1787829616.509469:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829616.509474:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829616.509478:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829616.509482:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:3.0:1787829616.509483:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829616.509490:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000001:3.0:1787829616.509491:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829616.509493:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829616.509499:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829616.509505:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829616.509507:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829616.509512:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3774)
00000800:00000200:0.1:1787829616.509521:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829616.509528:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.509532:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829616.509541:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829616.509548:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829616.509554:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829616.509557:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829616.509562:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829616.509567:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509579:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.509584:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509592:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509595:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829616.509603:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.509608:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829616.509611:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298437292
00000100:00000010:0.0:1787829616.509618:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc780.
00000100:00000001:0.0:1787829616.509626:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829616.509632:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298437292
00000100:00000010:0.0:1787829616.509635:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccde0.
00000100:00000001:0.0:1787829616.509639:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829616.509650:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829616.509654:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509697:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829616.509703:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616]  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829616.509714:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829616.509716:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829616.509718:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829616.509722:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829616.509724:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829616.509727:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829616.509731:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829616.509742:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509750:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829616.509752:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.509755:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.509758:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829616.509762:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509770:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829616.509776:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509785:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829616.509790:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.509792:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.509793:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.509796:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829616.509798:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829616.509802:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829616.509807:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829616.509810:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829616.509815:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829616.509826:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327361920:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829616.509831:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.509834:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.509837:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327361920/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509845:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829616.509849:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829616.509852:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829616.509855:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829616.509858:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829616.509860:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829616.509862:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829616.509865:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:0.0:1787829616.509870:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829616.509872:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829616.509874:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829616.509877:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829616.509879:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.509881:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.509891:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829616.509893:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.509896:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829616.509898:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829616.509899:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829616.509904:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509913:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829616.509915:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.509917:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829616.509919:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829616.509921:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829616.509925:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362048:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829616.509929:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829616.509931:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829616.509933:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.509935:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829616.509939:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829616.509942:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.509946:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829616.509949:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829616.509953:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000200:0.0:1787829616.509957:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362048, portal 10
00000100:00000001:0.0:1787829616.509959:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829616.509961:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:0.0:1787829616.509969:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.509977:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829616.509979:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829616.509982:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829616.509985:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327362048, offset 0
00000400:00000200:0.0:1787829616.509990:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829616.509994:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.509999:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829616.510003:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829616.510006:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829616.510009:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829616.510011:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829616.510015:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40600:40600:256:4294967295] 192.168.203.116@tcp LPNI seq info [40600:40600:8:4294967295]
00000400:00000200:0.0:1787829616.510020:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.510028:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829616.510034:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.510037:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829616.510042:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829616.510045:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829616.510050:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829616.510054:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829616.510058:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.510061:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829616.510064:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829616.510068:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829616.510071:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829616.510073:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829616.510076:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829616.510080:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510088:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.510092:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510100:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.510102:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829616.510119:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829616.510124:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829616.510128:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829616.510130:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298436619
00000100:00000001:0.0:1787829616.510134:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298437292
00000100:00000001:0.0:1787829616.510137:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298436619
00000100:00000001:0.0:1787829616.510139:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298437292
00000100:00000001:0.0:1787829616.510142:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829616.510144:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.510147:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829616.510152:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616]  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829616.510161:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829616.510163:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829616.510165:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829616.510168:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829616.510170:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829616.510173:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829616.510177:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829616.510187:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510194:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829616.510196:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510198:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.510200:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829616.510204:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510212:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829616.510216:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510224:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829616.510227:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510229:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.510231:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510233:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829616.510235:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829616.510238:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829616.510242:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829616.510244:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829616.510248:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829616.510258:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362048:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829616.510262:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829616.510266:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510274:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829616.510275:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.510278:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829616.510280:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829616.510282:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829616.510285:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362176:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829616.510289:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829616.510291:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829616.510293:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.510295:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829616.510299:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829616.510301:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.510306:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829616.510309:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829616.510312:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000200:0.0:1787829616.510316:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362176, portal 4
00000100:00000001:0.0:1787829616.510318:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829616.510320:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829616.510327:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510336:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829616.510338:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829616.510341:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829616.510344:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327362176, offset 0
00000400:00000200:0.0:1787829616.510348:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829616.510352:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.510357:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829616.510360:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829616.510363:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829616.510366:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829616.510368:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829616.510372:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40601:40601:256:4294967295] 192.168.203.116@tcp LPNI seq info [40601:40601:8:4294967295]
00000400:00000200:0.0:1787829616.510377:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.510384:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829616.510389:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.510393:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829616.510397:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829616.510400:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829616.510405:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829616.510423:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829616.510428:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.510430:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829616.510434:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829616.510437:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829616.510441:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829616.510443:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829616.510446:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829616.510450:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510459:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.510462:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510470:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.510471:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829616.510475:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829616.510478:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829616.510482:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829616.510484:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298436626
00000100:00000001:0.0:1787829616.510487:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298437292
00000100:00000001:0.0:1787829616.510490:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298436626
00000100:00000001:0.0:1787829616.510492:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298437292
00000100:00000001:0.0:1787829616.510494:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829616.510497:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.510499:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829616.510503:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616]  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829616.510511:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829616.510513:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829616.510515:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829616.510518:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829616.510520:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829616.510522:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829616.510525:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829616.510535:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510542:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829616.510544:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510546:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.510548:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829616.510553:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510561:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829616.510565:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510572:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829616.510576:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510577:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.510579:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510581:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829616.510582:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829616.510585:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829616.510588:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829616.510590:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829616.510594:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829616.510602:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362176:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829616.510606:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829616.510609:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510616:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829616.510617:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.510619:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829616.510621:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829616.510623:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829616.510626:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362304:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829616.510630:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829616.510632:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829616.510634:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829616.510636:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829616.510640:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829616.510642:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829616.510645:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829616.510647:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829616.510651:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000200:0.0:1787829616.510653:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362304, portal 4
00000100:00000001:0.0:1787829616.510674:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829616.510675:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:0.0:1787829616.510681:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510688:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829616.510690:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829616.510693:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829616.510696:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327362304, offset 0
00000400:00000200:0.0:1787829616.510701:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829616.510705:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.510708:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829616.510712:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829616.510715:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829616.510717:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829616.510719:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829616.510722:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40602:40602:256:4294967295] 192.168.203.116@tcp LPNI seq info [40602:40602:8:4294967295]
00000400:00000200:0.0:1787829616.510726:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829616.510733:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829616.510737:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.510740:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829616.510744:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829616.510747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829616.510751:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829616.510754:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829616.510757:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829616.510760:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829616.510763:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829616.510766:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829616.510769:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:0.0:1787829616.510770:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:0.0:1787829616.510773:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829616.510776:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510784:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.510787:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510795:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.510796:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829616.510799:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829616.510802:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829616.510806:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829616.510808:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298436632
00000100:00000001:0.0:1787829616.510811:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298437292
00000100:00000001:0.0:1787829616.510813:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298436632
00000100:00000001:0.0:1787829616.510816:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298437292
00000100:00000001:0.0:1787829616.510818:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829616.510820:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.510822:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829616.510826:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616]  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829616.510835:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829616.510836:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829616.510838:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829616.510841:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:0.0:1787829616.510843:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:0.0:1787829616.510845:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829616.510849:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829616.510858:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510865:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829616.510867:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510869:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.510871:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829616.510875:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510883:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829616.510887:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510895:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829616.510897:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510899:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829616.510900:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510901:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829616.510903:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829616.510906:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829616.510909:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829616.510911:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829616.510915:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829616.510924:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362304:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829616.510928:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510930:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.510934:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327362048/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510940:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829616.510942:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829616.510944:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829616.510947:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829616.510949:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829616.510950:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829616.510953:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829616.510955:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829616.510957:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829616.510959:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829616.510960:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829616.510963:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829616.510965:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.510967:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.510971:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327362176/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.510978:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829616.510979:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829616.510981:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829616.510983:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829616.510984:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829616.510986:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829616.510988:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829616.510990:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829616.510992:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829616.510993:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829616.510995:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829616.510997:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829616.510999:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.511001:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829616.511004:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327362304/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829616.511012:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829616.511014:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829616.511015:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829616.511018:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829616.511019:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829616.511021:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829616.511023:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829616.511024:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829616.511026:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829616.511027:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829616.511028:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829616.511030:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829616.511031:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.511033:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829616.511037:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829616.511038:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829616.511040:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829616.511042:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829617.020749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829617.020757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829617.020759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829617.020764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829617.020767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829617.020771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829617.020773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829617.020776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829617.020778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829617.212806:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298437292, at 4298437312
00000100:00000040:3.0:1787829617.212808:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298437292, at 4298437312
00000100:00000010:2.0:1787829617.212825:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000100:00000010:3.0:1787829617.212830:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00010000:00000001:0.0:1787829617.340765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829617.340772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829617.340778:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829617.340779:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829617.340782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829617.340783:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829617.340842:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829617.340847:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829617.340851:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829618.044740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829618.044746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829618.044748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829618.044752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829618.044756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829618.044758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829618.044761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829618.044763:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829618.044765:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829618.364737:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829618.364742:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829618.364747:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829618.364748:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829618.364751:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829618.364752:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829618.364786:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829618.364790:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829618.364793:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829619.068788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829619.068795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829619.068797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829619.068802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829619.068807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829619.068812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829619.068815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829619.068818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829619.068822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829619.388777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829619.388786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829619.388793:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829619.388794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829619.388797:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829619.388800:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829619.388803:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829619.388807:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829619.388811:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829619.388814:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829619.388816:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829619.388819:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829619.388846:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829619.388850:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829619.388854:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829620.092748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829620.092754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829620.092755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829620.092759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829620.092762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829620.092765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829620.092768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829620.092770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829620.092772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829620.412809:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829620.412815:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829620.412821:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829620.412822:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829620.412826:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829620.412827:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829620.412885:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829620.412889:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829620.412891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829621.116769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829621.116775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829621.116776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829621.116781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829621.116783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829621.116786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829621.116796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829621.116799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829621.116801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829621.436785:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829621.436793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829621.436799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829621.436801:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829621.436804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829621.436806:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829621.436860:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829621.436863:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829621.436865:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829621.628749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829621.628757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829621.628759:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829621.628762:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829621.628766:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829621.628771:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3769
00000400:00000200:2.0:1787829621.628782:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829621.628788:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829621.628790:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829621.628793:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.628795:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829621.628798:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.628799:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829621.628803:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829621.628804:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829621.628806:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829621.628810:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829621.628811:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829621.628813:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829621.628815:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.628816:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829621.628817:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.628819:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829621.628821:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829621.628824:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829621.628825:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829621.628827:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829621.628830:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829621.628832:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.628833:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829621.628835:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829621.628842:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829621.628845:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829621.628848:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829621.628849:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.628851:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829621.628853:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829621.628856:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829621.628862:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829621.628867:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829621.628874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829621.628882:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829621.628885:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829621.628890:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829621.628912:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829621.628915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829621.628919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829621.628920:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829621.628922:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829621.628925:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829621.628928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3769
00000400:00000200:2.0:1787829621.628933:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829621.628937:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829621.628940:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829621.628943:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787829621.628944:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829621.628949:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829621.628949:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829621.628953:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829621.628954:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.628957:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829621.628959:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829621.628961:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000040:0.0:1787829621.628963:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829621.628963:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829621.628967:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829621.628971:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829621.628976:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829621.628983:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787829621.628983:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829621.628986:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.628987:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829621.628991:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829621.628992:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787829621.628993:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829621.628995:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1787829621.628996:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829621.629002:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829621.629003:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:0.0:1787829621.629009:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327362432:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787829621.629010:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829621.629014:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829621.629019:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829621.629020:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829621.629024:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829621.629024:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629028:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629028:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829621.629031:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829621.629032:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:2.0:1787829621.629036:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629038:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829621.629039:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829621.629042:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829621.629043:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787829621.629047:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829621.629049:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829621.629050:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829621.629051:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629054:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1787829621.629056:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
02000000:00000001:2.0:1787829621.629056:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829621.629059:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829621.629064:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
02000000:00000010:2.0:1787829621.629065:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829621.629085:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829621.629086:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362432, portal 25
00000100:00000001:0.0:1787829621.629091:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00100000:2.0:1787829621.629091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829621.629093:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787829621.629102:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1787829621.629104:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829621.629105:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829621.629113:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829621.629118:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829621.629126:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629127:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829621.629129:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00080000:2.0:1787829621.629131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829621.629134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:0.0:1787829621.629135:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327362432, offset 0
00000100:00000001:2.0:1787829621.629136:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829621.629139:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829621.629142:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829621.629145:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829621.629146:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3769
00000400:00000200:2.0:1787829621.629151:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829621.629154:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829621.629157:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829621.629158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829621.629163:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629165:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829621.629167:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:2.0:1787829621.629170:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629173:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829621.629175:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1787829621.629178:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829621.629180:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:0.0:1787829621.629182:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:2.0:1787829621.629183:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829621.629187:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829621.629188:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1787829621.629190:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829621.629193:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000001:2.0:1787829621.629194:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829621.629199:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40603:40603:256:4294967295] 192.168.203.116@tcp LPNI seq info [40603:40603:8:4294967295]
00000100:00000040:2.0:1787829621.629201:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629206:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829621.629208:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829621.629210:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629213:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829621.629214:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829621.629218:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629223:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829621.629227:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1787829621.629228:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829621.629232:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829621.629235:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629238:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829621.629239:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000008:00100000:2.0:1787829621.629240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829621.629245:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629246:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829621.629249:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00000010:2.0:1787829621.629250:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829621.629255:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829621.629256:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829621.629258:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629261:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.0:1787829621.629262:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829621.629263:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829621.629265:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829621.629271:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829621.629276:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829621.629277:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:2.0:1787829621.629284:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829621.629289:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829621.629290:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829621.629293:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829621.629298:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829621.629300:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829621.629301:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829621.629302:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829621.629305:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829621.629307:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829621.629309:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829621.629311:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829621.629312:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3769
00000400:00000200:2.0:1787829621.629318:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829621.629319:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298442412
00000400:00000200:2.0:1787829621.629322:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829621.629325:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000010:0.0:1787829621.629327:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd20.
00000100:00000040:2.0:1787829621.629328:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629332:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829621.629338:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787829621.629338:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629340:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829621.629344:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829621.629346:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829621.629347:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298442412
00000020:00000001:2.0:1787829621.629349:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829621.629353:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000010:0.0:1787829621.629354:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc660.
00000020:00000001:2.0:1787829621.629355:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829621.629359:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000001:2.0:1787829621.629359:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829621.629365:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629367:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829621.629368:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829621.629369:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829621.629370:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829621.629372:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1787829621.629373:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829621.629378:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629379:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829621.629384:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829621.629386:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829621.629387:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829621.629391:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787829621.629392:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629398:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829621.629399:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829621.629401:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629403:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829621.629404:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629409:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000010:2.0:1787829621.629410:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829621.629415:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829621.629418:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787829621.629419:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829621.629422:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629423:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829621.629424:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829621.629428:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1787829621.629428:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829621.629429:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829621.629434:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829621.629438:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829621.629439:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829621.629444:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829621.629471:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829621.629472:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829621.629475:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829621.629476:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629480:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829621.629485:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787829621.629485:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829621.629492:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829621.629500:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362560:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829621.629504:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629506:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829621.629511:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1787829621.629511:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3779)
02000000:00000001:0.0:1787829621.629513:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629518:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829621.629523:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829621.629526:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829621.629534:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829621.629539:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708d68.
00000400:00000010:0.0:1787829621.629544:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829621.629547:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362560, portal 10
00000100:00000001:0.0:1787829621.629550:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829621.629551:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:0.0:1787829621.629561:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829621.629569:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829621.629572:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829621.629575:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829621.629578:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327362560, offset 0
00000400:00000200:0.0:1787829621.629582:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829621.629587:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829621.629591:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829621.629594:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829621.629597:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829621.629600:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829621.629603:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829621.629607:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40604:40604:255:4294967295] 192.168.203.116@tcp LPNI seq info [40604:40604:7:4294967295]
00000400:00000200:0.0:1787829621.629611:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829621.629619:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829621.629624:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829621.629628:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829621.629633:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829621.629637:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829621.629640:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829621.629644:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829621.629647:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829621.629649:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298441738
00000100:00000001:0.0:1787829621.629652:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298442412
00000100:00000001:0.0:1787829621.629676:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298441739
00000100:00000001:0.0:1787829621.629680:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298442412
00000100:00000001:0.0:1787829621.629683:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829621.629685:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629688:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829621.629692:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829621.629700:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829621.629702:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629704:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829621.629706:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829621.629708:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829621.629711:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362688:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829621.629721:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829621.629723:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829621.629725:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629727:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829621.629731:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829621.629733:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829621.629737:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829621.629739:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829621.629743:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829621.629746:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362688, portal 4
00000100:00000001:0.0:1787829621.629748:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829621.629750:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:0.0:1787829621.629755:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829621.629762:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829621.629765:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829621.629768:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fefd48.
00000100:00000200:0.0:1787829621.629770:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327362688, offset 0
00000400:00000200:0.0:1787829621.629775:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829621.629779:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829621.629783:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829621.629786:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829621.629788:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829621.629791:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829621.629793:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829621.629796:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40605:40605:254:4294967295] 192.168.203.116@tcp LPNI seq info [40605:40605:6:4294967295]
00000400:00000200:0.0:1787829621.629799:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829621.629805:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829621.629809:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829621.629813:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7f00.
00000800:00000200:0.0:1787829621.629817:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829621.629821:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829621.629824:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829621.629826:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829621.629829:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829621.629831:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298441746
00000100:00000001:0.0:1787829621.629833:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298442412
00000100:00000001:0.0:1787829621.629835:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298441746
00000100:00000001:0.0:1787829621.629838:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298442412
00000100:00000001:0.0:1787829621.629840:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829621.629842:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629845:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.629847:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829621.629850:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829621.629851:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829621.629854:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829621.629855:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829621.629856:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829621.629861:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829621.629868:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829621.629870:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629872:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829621.629874:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829621.629877:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829621.629880:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362816:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829621.629883:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829621.629885:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829621.629887:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829621.629889:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829621.629893:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829621.629895:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829621.629898:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829621.629901:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829621.629905:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829621.629908:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362816, portal 4
00000100:00000001:0.0:1787829621.629910:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829621.629911:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:0.0:1787829621.629918:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829621.629925:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829621.629928:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829621.629930:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef330.
00000100:00000200:0.0:1787829621.629933:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327362816, offset 0
00000400:00000200:0.0:1787829621.629938:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829621.629942:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829621.629946:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829621.629949:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829621.629952:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829621.629955:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829621.629957:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829621.629960:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40606:40606:253:4294967295] 192.168.203.116@tcp LPNI seq info [40606:40606:5:4294967295]
00000400:00000200:0.0:1787829621.629965:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829621.629971:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829621.629976:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829621.629980:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7000.
00000800:00000200:0.0:1787829621.629984:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829621.629988:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829621.629990:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829621.629993:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829621.629996:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829621.629999:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298441751
00000100:00000001:0.0:1787829621.630002:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298442412
00000100:00000001:0.0:1787829621.630004:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298441751
00000100:00000001:0.0:1787829621.630006:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298442412
00000100:00000001:0.0:1787829621.630008:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829621.630010:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.630012:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.630014:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829621.630017:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829621.630018:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829621.630020:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829621.630021:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829621.630023:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829621.630025:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829622.140762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829622.140767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829622.140769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829622.140773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829622.140775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829622.140778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829622.140780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829622.140782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829622.140784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829622.332709:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298442412, at 4298442432
00000100:00000010:3.0:1787829622.332727:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000040:2.0:1787829622.332770:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298442412, at 4298442432
00000100:00000010:2.0:1787829622.332783:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00010000:00000001:0.0:1787829622.460787:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829622.460795:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829622.460801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829622.460802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829622.460806:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829622.460808:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829622.460828:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829622.460831:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829622.460833:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829623.164809:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829623.164818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829623.164820:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829623.164828:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829623.164833:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829623.164838:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829623.164840:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829623.164843:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829623.164845:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829623.484749:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829623.484757:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829623.484764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829623.484765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829623.484769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829623.484771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829623.484827:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829623.484833:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829623.484836:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829624.188795:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829624.188801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829624.188802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829624.188806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829624.188808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829624.188811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.188813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829624.188815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.188817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829624.508797:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829624.508801:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829624.508805:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508811:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829624.508813:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508815:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508817:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508821:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829624.508822:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829624.508823:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829624.508825:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508826:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829624.508827:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508828:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508830:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829624.508834:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829624.508836:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829624.508882:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829624.508887:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829624.508891:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1787829624.701011:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:1.0:1787829624.701088:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1787829624.701100:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829624.701105:0:2334:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:1.0:1787829624.701112:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829624.701117:0:2334:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1787829624.701123:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829624.701128:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:1.0:1787829624.701136:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829624.701141:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829624.701150:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000400:00000010:1.0:1787829624.701153:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:1.0:1787829624.701159:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829624.701176:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829624.701205:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829624.701211:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829624.701221:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829624.701223:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829624.701241:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829624.701245:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:1.0:1787829624.701250:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829624.701256:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000001:0.0:1787829624.701259:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1787829624.701261:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:1.0:1787829624.701268:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829624.701271:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829624.701272:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829624.701279:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000200:1.0:1787829624.701280:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829624.701292:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1787829624.701294:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829624]  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829624.701299:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829624.701323:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829624.701325:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000001:0.0:1787829624.701327:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000800:00000100:1.0:1787829624.701332:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000400:00000001:0.0:1787829624.701333:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000800:00000010:1.0:1787829624.701338:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000010:0.0:1787829624.701340:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:1.0:1787829624.701343:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829624.701348:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787829624.701349:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000200:1.0:1787829624.701351:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefd48
00000400:00000010:0.0:1787829624.701354:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000400:00000010:1.0:1787829624.701356:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefd48.
00000100:00000001:0.0:1787829624.701362:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829624.701362:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829624.701370:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829624.701372:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829624.701392:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829624.701399:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829624.701406:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829624.701408:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829624.701419:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000100:00000200:0.0:1787829624.701420:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1787829624.701421:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000200:1.0:1787829624.701427:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829624.701435:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000001:0.0:1787829624.701441:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000200:1.0:1787829624.701441:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef330
00000400:00000010:1.0:1787829624.701444:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef330.
00000100:00000001:1.0:1787829624.701447:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829624.701451:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:1.0:1787829624.701455:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701457:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701462:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829624.701469:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1787829624.701473:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829624.701478:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701498:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829624.701499:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829624.701504:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787829624.701515:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701526:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829624.701532:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701534:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701536:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701539:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829624.701542:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829624.701547:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829624.701554:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829624.701557:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829624.701563:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829624.701573:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327362432:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829624.701578:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829624.701581:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829624.701583:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327362432/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701590:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829624.701595:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829624.701601:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829624.701606:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829624.701610:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829624.701612:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829624.701615:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829624.701619:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829624.701623:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829624.701625:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829624.701628:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:0.0:1787829624.701632:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829624.701633:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701635:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829624.701637:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829624.701639:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829624.701641:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829624.701646:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829624]  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829624.701699:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829624.701702:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829624.701705:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829624.701709:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829624.701712:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829624.701716:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829624.701720:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829624.701731:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701740:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829624.701742:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701745:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701746:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829624.701750:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701758:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829624.701763:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701771:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829624.701774:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701776:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701777:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701780:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829624.701782:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829624.701785:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829624.701790:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829624.701793:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829624.701799:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829624.701809:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362816:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829624.701814:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829624.701818:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829624]  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829624.701827:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829624.701829:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829624.701831:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708d68.
00000400:00000200:0.0:1787829624.701834:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829624.701837:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829624.701839:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829624.701843:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829624.701852:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701859:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829624.701860:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701863:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701864:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829624.701869:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701878:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829624.701882:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701890:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829624.701892:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701895:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701897:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701898:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829624.701901:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829624.701904:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829624.701907:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829624.701910:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829624.701914:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829624.701923:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362560:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829624.701928:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829624.701932:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829624]  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829624.701940:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829624.701941:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829624.701943:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829624.701946:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829624.701948:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829624.701951:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829624.701954:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829624.701962:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701969:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829624.701970:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.701972:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.701974:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829624.701978:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.701991:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829624.701995:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.702002:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829624.702004:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702006:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829624.702007:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702008:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829624.702010:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829624.702013:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829624.702017:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829624.702020:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829624.702024:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829624.702032:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327362688:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829624.702038:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702040:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829624.702045:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327362816/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.702066:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829624.702068:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829624.702071:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829624.702075:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829624.702077:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829624.702079:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829624.702081:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829624.702083:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829624.702086:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829624.702088:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829624.702089:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:0.0:1787829624.702093:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829624.702094:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702096:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829624.702101:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327362560/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.702108:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829624.702109:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829624.702111:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829624.702113:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829624.702115:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829624.702116:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829624.702118:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829624.702120:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829624.702123:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829624.702124:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829624.702125:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:0.0:1787829624.702128:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829624.702130:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702131:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829624.702135:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327362688/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829624.702142:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829624.702143:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829624.702145:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829624.702147:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829624.702149:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829624.702150:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829624.702152:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829624.702154:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829624.702157:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829624.702158:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829624.702160:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:0.0:1787829624.702163:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829624.702164:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702167:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829624.702183:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829624.702187:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829624.702191:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829624.702192:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829625.212871:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829625.212880:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829625.212882:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829625.212887:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829625.212893:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829625.212897:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829625.212900:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829625.212902:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829625.212904:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829625.532759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829625.532766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829625.532772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829625.532773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829625.532776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829625.532777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829625.532798:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829625.532800:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829625.532803:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829626.236736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829626.236743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829626.236746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829626.236751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829626.236754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829626.236758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829626.236772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829626.236776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829626.236780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829626.556796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829626.556805:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829626.556823:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829626.556825:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829626.556829:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829626.556831:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829626.556909:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829626.556913:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829626.556916:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829626.748848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829626.748856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829626.748858:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829626.748861:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829626.748864:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829626.748870:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3774
00000400:00000200:3.0:1787829626.748877:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829626.748881:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829626.748884:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829626.748886:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.748888:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829626.748890:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.748892:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829626.748895:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829626.748897:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829626.748898:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829626.748901:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829626.748902:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829626.748904:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829626.748906:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.748907:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829626.748908:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.748910:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829626.748912:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829626.748915:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829626.748916:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829626.748919:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829626.748922:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829626.748924:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.748926:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829626.748928:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829626.748933:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829626.748936:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829626.748939:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829626.748941:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.748943:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829626.748945:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829626.748949:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829626.748957:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829626.748964:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829626.748973:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829626.748984:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829626.748989:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829626.748994:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829626.749021:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829626.749025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829626.749029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829626.749030:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829626.749032:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829626.749049:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829626.749053:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3774
00000400:00000200:3.0:1787829626.749063:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829626.749067:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829626.749070:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829626.749077:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749080:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829626.749084:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749086:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829626.749089:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829626.749093:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1787829626.749094:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829626.749099:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829626.749105:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829626.749108:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829626.749110:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829626.749114:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787829626.749115:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829626.749121:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829626.749122:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829626.749131:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749133:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829626.749137:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:0.0:1787829626.749143:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829626.749145:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749154:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829626.749164:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829626.749174:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787829626.749175:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829626.749182:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829626.749184:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.749191:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829626.749192:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1787829626.749198:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829626.749203:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1787829626.749204:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.749216:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.749217:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829626.749230:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829626.749236:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829626.749238:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829626.749240:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.749241:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829626.749243:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829626.749245:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829626.749250:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327362944:192.168.203.116@tcp:250:kworker.0
02000000:00000010:3.0:1787829626.749253:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:0.0:1787829626.749271:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829626.749272:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.749280:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829626.749288:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829626.749288:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829626.749300:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829626.749311:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829626.749315:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829626.749316:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
00000100:00000040:3.0:1787829626.749326:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829626.749328:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.749348:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829626.749352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829626.749355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829626.749357:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829626.749359:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829626.749360:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829626.749370:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1787829626.749372:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00080000:3.0:1787829626.749376:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3774
00000400:00000200:3.0:1787829626.749384:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829626.749386:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000400:00000200:3.0:1787829626.749393:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829626.749397:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1787829626.749400:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327362944, portal 25
00000100:00000040:3.0:1787829626.749402:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829626.749407:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787829626.749410:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829626.749414:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:3.0:1787829626.749419:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749426:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829626.749431:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829626.749432:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829626.749438:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829626.749444:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829626.749453:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000001:0.0:1787829626.749464:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:3.0:1787829626.749465:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:0.0:1787829626.749477:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829626.749478:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829626.749491:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749494:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829626.749498:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000040:3.0:1787829626.749499:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749506:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829626.749509:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327362944, offset 0
00000020:00000040:3.0:1787829626.749512:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829626.749520:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749527:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.749530:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829626.749533:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829626.749541:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829626.749544:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.749547:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829626.749548:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:3.0:1787829626.749552:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829626.749558:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.749564:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829626.749566:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829626.749577:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000400:00000200:0.0:1787829626.749578:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1787829626.749587:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829626.749595:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829626.749599:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:3.0:1787829626.749599:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.749606:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829626.749609:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829626.749612:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787829626.749616:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.749620:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829626.749627:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40607:40607:256:4294967295] 192.168.203.116@tcp LPNI seq info [40607:40607:8:4294967295]
00000400:00000200:0.0:1787829626.749632:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.749646:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829626.749705:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.749716:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829626.749727:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829626.749733:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829626.749741:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829626.749749:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
02000000:00000010:3.0:1787829626.749752:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000800:00000100:0.0:1787829626.749761:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749763:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829626.749766:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829626.749776:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:3.0:1787829626.749778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829626.749798:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787829626.749798:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829626.749810:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.749820:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000100:00000040:3.0:1787829626.749821:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829626.749827:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829626.749839:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787829626.749845:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829626.749850:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829626.749853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829626.749870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829626.749876:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829626.749881:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829626.749888:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829626.749889:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1787829626.749896:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3774
00000100:00000040:0.0:1787829626.749902:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787829626.749919:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829626.749929:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829626.749933:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829626.749935:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829626.749940:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:3.0:1787829626.749945:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829626.749953:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829626.749954:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829626.749964:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829626.749972:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829626.749975:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829626.749979:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298447533
00000020:00000001:3.0:1787829626.749983:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829626.749987:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829626.749990:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:0.0:1787829626.749991:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc5a0.
00000020:00000040:3.0:1787829626.749994:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829626.750000:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829626.750004:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000020:00000001:3.0:1787829626.750004:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829626.750016:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829626.750020:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298447533
00000100:00000001:3.0:1787829626.750021:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829626.750032:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:0.0:1787829626.750033:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf60.
00000100:00000001:3.0:1787829626.750053:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829626.750057:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000040:3.0:1787829626.750061:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829626.750066:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787829626.750073:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829626.750076:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.750081:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.750085:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1787829626.750088:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829626.750095:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1787829626.750096:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626]  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829626.750105:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.750116:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829626.750124:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829626.750131:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829626.750132:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787829626.750135:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829626.750137:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829626.750142:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000010:3.0:1787829626.750144:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000400:00000200:0.0:1787829626.750150:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000020:00000040:3.0:1787829626.750152:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000010:0.0:1787829626.750159:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
02000000:00000001:3.0:1787829626.750162:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829626.750167:0:2346:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1787829626.750167:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829626.750177:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:0.0:1787829626.750179:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829626.750182:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829626.750185:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829626.750198:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829626.750204:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829626.750218:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829626.750220:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829626.750232:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829626.750239:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829626.750257:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750258:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829626.750262:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750270:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.750274:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829626.750286:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829626.750289:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829626.750294:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3784)
00000100:00000001:0.0:1787829626.750309:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829626.750335:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750345:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829626.750349:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750352:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.750353:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750357:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829626.750359:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829626.750364:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829626.750370:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829626.750373:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829626.750379:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829626.750390:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327362944:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829626.750397:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750400:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.750404:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327362944/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750412:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829626.750415:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829626.750421:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829626.750425:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829626.750428:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829626.750431:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829626.750434:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829626.750437:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:0.0:1787829626.750441:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829626.750444:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829626.750446:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:0.0:1787829626.750451:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829626.750453:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750456:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750465:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829626.750467:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.750470:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829626.750473:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829626.750475:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829626.750480:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750489:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829626.750491:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.750494:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829626.750497:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829626.750500:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829626.750504:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363072:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829626.750508:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829626.750511:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829626.750513:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.750516:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829626.750520:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829626.750523:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.750527:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829626.750531:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829626.750536:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829626.750540:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363072, portal 10
00000100:00000001:0.0:1787829626.750543:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829626.750545:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:0.0:1787829626.750554:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750561:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829626.750564:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829626.750567:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829626.750570:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327363072, offset 0
00000400:00000200:0.0:1787829626.750576:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829626.750581:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.750588:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829626.750592:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829626.750595:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829626.750598:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829626.750601:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829626.750605:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40608:40608:256:4294967295] 192.168.203.116@tcp LPNI seq info [40608:40608:8:4294967295]
00000400:00000200:0.0:1787829626.750610:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.750618:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829626.750624:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.750629:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829626.750635:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829626.750638:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829626.750644:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829626.750647:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829626.750652:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.750654:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829626.750658:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829626.750662:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829626.750666:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829626.750668:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829626.750700:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829626.750705:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750738:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.750742:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750750:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.750752:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829626.750756:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829626.750760:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829626.750764:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829626.750767:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298446860
00000100:00000001:0.0:1787829626.750770:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298447533
00000100:00000001:0.0:1787829626.750773:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298446860
00000100:00000001:0.0:1787829626.750776:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298447533
00000100:00000001:0.0:1787829626.750779:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829626.750781:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.750783:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829626.750788:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626]  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829626.750814:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829626.750816:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829626.750817:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829626.750821:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829626.750823:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829626.750826:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829626.750830:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829626.750841:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750849:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829626.750852:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750854:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.750856:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829626.750861:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750870:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829626.750875:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750884:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829626.750887:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750889:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.750891:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.750893:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829626.750895:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829626.750899:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829626.750903:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829626.750906:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829626.750912:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829626.750922:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363072:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829626.750927:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829626.750932:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.750940:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829626.750942:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.750945:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829626.750948:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829626.750950:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829626.750954:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363200:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829626.750959:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829626.750961:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829626.750963:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.750966:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829626.750970:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829626.750972:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.750977:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829626.750980:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829626.750984:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829626.750988:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363200, portal 4
00000100:00000001:0.0:1787829626.750991:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829626.750993:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:0.0:1787829626.751000:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751008:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829626.751011:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829626.751013:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829626.751017:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327363200, offset 0
00000400:00000200:0.0:1787829626.751022:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829626.751027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.751031:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829626.751049:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829626.751053:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829626.751056:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829626.751059:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829626.751063:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40609:40609:256:4294967295] 192.168.203.116@tcp LPNI seq info [40609:40609:8:4294967295]
00000400:00000200:0.0:1787829626.751068:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.751093:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829626.751098:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.751102:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829626.751108:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829626.751112:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829626.751116:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829626.751120:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829626.751124:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.751127:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829626.751130:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829626.751134:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829626.751139:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829626.751141:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829626.751144:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829626.751148:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751156:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.751160:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751168:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.751170:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829626.751173:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829626.751176:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829626.751181:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829626.751183:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298446867
00000100:00000001:0.0:1787829626.751186:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298447533
00000100:00000001:0.0:1787829626.751189:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298446867
00000100:00000001:0.0:1787829626.751192:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298447533
00000100:00000001:0.0:1787829626.751195:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829626.751197:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.751199:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829626.751204:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626]  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829626.751213:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829626.751215:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829626.751217:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829626.751220:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829626.751222:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829626.751225:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829626.751228:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829626.751238:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751247:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829626.751248:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751250:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.751253:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829626.751257:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751265:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829626.751270:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751279:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829626.751282:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751284:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.751286:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751288:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829626.751290:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829626.751294:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829626.751298:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829626.751301:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829626.751306:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829626.751324:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363200:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829626.751329:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829626.751334:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751342:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829626.751344:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.751347:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829626.751349:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829626.751352:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829626.751355:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363328:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829626.751359:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829626.751362:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829626.751363:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829626.751366:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829626.751369:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829626.751372:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829626.751376:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829626.751379:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829626.751383:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829626.751386:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363328, portal 4
00000100:00000001:0.0:1787829626.751389:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829626.751391:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829626.751397:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751405:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829626.751408:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829626.751410:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829626.751413:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327363328, offset 0
00000400:00000200:0.0:1787829626.751418:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829626.751423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.751427:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829626.751431:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829626.751434:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829626.751437:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829626.751440:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829626.751443:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40610:40610:256:4294967295] 192.168.203.116@tcp LPNI seq info [40610:40610:8:4294967295]
00000400:00000200:0.0:1787829626.751448:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829626.751456:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829626.751461:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.751465:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829626.751469:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829626.751473:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829626.751478:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829626.751481:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829626.751486:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829626.751488:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829626.751492:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829626.751496:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829626.751499:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829626.751501:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829626.751504:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829626.751509:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751518:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.751522:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751529:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.751531:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829626.751534:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829626.751537:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829626.751541:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829626.751544:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298446872
00000100:00000001:0.0:1787829626.751547:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298447533
00000100:00000001:0.0:1787829626.751550:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298446872
00000100:00000001:0.0:1787829626.751552:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298447533
00000100:00000001:0.0:1787829626.751555:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829626.751558:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.751560:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829626.751565:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626]  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829626.751574:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829626.751576:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829626.751578:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829626.751580:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829626.751582:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829626.751585:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829626.751589:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829626.751599:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751608:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829626.751610:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751612:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.751613:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829626.751618:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751627:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829626.751631:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751639:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829626.751642:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751644:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829626.751646:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751648:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829626.751650:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829626.751653:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829626.751657:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829626.751660:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829626.751665:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829626.751694:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363328:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829626.751699:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751702:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.751706:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327363072/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751714:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829626.751716:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829626.751719:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829626.751722:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829626.751724:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829626.751725:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829626.751728:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829626.751731:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:0.0:1787829626.751734:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829626.751736:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829626.751737:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:0.0:1787829626.751741:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829626.751742:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751745:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.751749:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327363200/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751764:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829626.751766:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829626.751768:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829626.751770:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829626.751772:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829626.751773:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829626.751792:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829626.751794:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:0.0:1787829626.751797:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829626.751798:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829626.751800:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829626.751803:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829626.751804:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751807:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829626.751810:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327363328/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829626.751819:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829626.751820:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829626.751822:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829626.751825:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829626.751826:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829626.751828:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829626.751830:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829626.751832:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:0.0:1787829626.751834:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829626.751835:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829626.751837:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829626.751839:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829626.751841:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751843:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829626.751848:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829626.751850:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829626.751852:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829626.751854:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829627.260737:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829627.260745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829627.260747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829627.260753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829627.260758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829627.260763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829627.260766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829627.260770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829627.260774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829627.452746:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298447533, at 4298447552
00000100:00000040:2.0:1787829627.452760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298447533, at 4298447552
00000100:00000010:3.0:1787829627.452762:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000100:00000010:2.0:1787829627.452776:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00010000:00000001:0.0:1787829627.580759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829627.580766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829627.580772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829627.580773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829627.580776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829627.580778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829627.580814:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829627.580817:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829627.580821:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829628.284771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829628.284781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829628.284784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829628.284792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829628.284797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829628.284802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829628.284805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829628.284810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829628.284813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829628.604808:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829628.604816:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829628.604824:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829628.604826:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829628.604830:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829628.604831:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829628.604861:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829628.604864:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829628.604868:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829629.308723:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829629.308733:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829629.308735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829629.308743:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829629.308747:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829629.308752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829629.308756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829629.308760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829629.308764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829629.628763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829629.628770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829629.628776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829629.628778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829629.628780:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829629.628783:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829629.628786:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829629.628789:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829629.628791:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829629.628793:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829629.628794:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829629.628796:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829629.628846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829629.628850:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829629.628853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829630.332734:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829630.332744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829630.332747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829630.332755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829630.332760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829630.332765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829630.332768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829630.332772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829630.332776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829630.652800:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829630.652806:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829630.652811:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829630.652812:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829630.652815:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829630.652816:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829630.652862:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829630.652868:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829630.652872:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829631.356797:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829631.356806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829631.356809:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829631.356816:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829631.356821:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829631.356827:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829631.356830:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829631.356834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829631.356838:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829631.676760:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829631.676771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829631.676779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829631.676781:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829631.676785:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829631.676786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829631.676816:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829631.676819:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829631.676823:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829631.868796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829631.868805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829631.868807:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829631.868810:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829631.868813:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829631.868819:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3779
00000400:00000200:1.0:1787829631.868827:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829631.868832:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829631.868835:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829631.868837:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.868839:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829631.868841:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.868843:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829631.868846:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829631.868848:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829631.868850:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829631.868853:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829631.868854:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829631.868864:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829631.868867:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.868869:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829631.868874:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.868875:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829631.868878:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829631.868880:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829631.868882:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829631.868884:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829631.868887:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829631.868890:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.868891:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829631.868893:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829631.868901:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829631.868903:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829631.868907:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829631.868908:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.868910:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829631.868912:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829631.868915:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829631.868923:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829631.868929:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829631.868936:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829631.868944:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829631.868947:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829631.868951:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829631.868968:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829631.868970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829631.868972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829631.868974:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829631.868975:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829631.868977:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829631.868980:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3779
00000400:00000200:1.0:1787829631.868984:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829631.868990:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829631.868992:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829631.868994:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.868995:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829631.868998:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829631.869008:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829631.869012:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829631.869016:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829631.869017:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1787829631.869018:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829631.869026:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829631.869027:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787829631.869035:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829631.869039:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829631.869043:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829631.869046:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829631.869050:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869058:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829631.869067:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869076:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829631.869081:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1787829631.869081:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829631.869085:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829631.869091:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869094:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.869095:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829631.869097:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829631.869099:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829631.869103:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829631.869104:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829631.869117:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829631.869123:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327363456:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829631.869124:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829631.869132:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829631.869134:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:1.0:1787829631.869142:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.869146:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829631.869147:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.869157:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829631.869158:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
02000000:00000001:0.0:1787829631.869161:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:1.0:1787829631.869162:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829631.869166:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829631.869167:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869169:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:0.0:1787829631.869171:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:1.0:1787829631.869172:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829631.869177:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829631.869178:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829631.869193:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000400:00000200:0.0:1787829631.869197:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829631.869201:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829631.869203:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00100000:1.0:1787829631.869211:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829631.869212:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000001:1.0:1787829631.869225:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:0.0:1787829631.869226:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363456, portal 25
00000100:00000001:1.0:1787829631.869227:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.869233:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:1.0:1787829631.869235:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.869236:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:0.0:1787829631.869254:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829631.869256:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829631.869267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829631.869277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829631.869285:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829631.869287:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829631.869290:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829631.869294:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3779
00000100:00000001:0.0:1787829631.869296:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787829631.869299:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829631.869304:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000040:0.0:1787829631.869307:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829631.869308:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829631.869314:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869315:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829631.869317:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000040:1.0:1787829631.869317:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869327:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829631.869329:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327363456, offset 0
00000020:00000001:1.0:1787829631.869329:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829631.869331:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829631.869336:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829631.869339:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829631.869342:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829631.869347:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000001:1.0:1787829631.869353:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829631.869358:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869359:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829631.869360:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829631.869367:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829631.869369:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829631.869373:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829631.869376:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829631.869379:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829631.869383:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829631.869389:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829631.869393:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1787829631.869394:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829631.869397:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869399:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829631.869401:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829631.869406:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000008:00000001:1.0:1787829631.869407:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869412:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829631.869416:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829631.869419:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829631.869421:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787829631.869423:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829631.869425:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869428:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829631.869429:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829631.869435:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:1.0:1787829631.869435:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829631.869444:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000400:00000200:0.0:1787829631.869451:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40611:40611:256:4294967295] 192.168.203.116@tcp LPNI seq info [40611:40611:8:4294967295]
00000100:00000001:1.0:1787829631.869451:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829631.869462:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829631.869472:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829631.869474:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829631.869477:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829631.869481:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829631.869491:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829631.869498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829631.869499:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1787829631.869501:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829631.869506:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829631.869508:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.0:1787829631.869512:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869517:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:0.0:1787829631.869519:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00080000:1.0:1787829631.869524:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3779
00000800:00000200:0.0:1787829631.869530:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:1.0:1787829631.869538:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.0:1787829631.869539:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000400:00000200:1.0:1787829631.869555:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:0.1:1787829631.869556:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829631.869566:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00080000:1.0:1787829631.869568:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000100:0.0:1787829631.869574:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.869579:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000100:00000040:1.0:1787829631.869579:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829631.869588:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829631.869591:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1787829631.869592:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829631.869606:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787829631.869608:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829631.869613:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000020:00000001:1.0:1787829631.869614:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829631.869619:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000010:0.0:1787829631.869621:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000020:00000001:1.0:1787829631.869625:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829631.869629:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:1.0:1787829631.869629:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829631.869636:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829631.869639:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829631.869640:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829631.869692:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.869697:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.869703:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.869705:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787829631.869711:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829631.869712:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869721:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829631.869725:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829631.869733:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829631.869734:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298452653
00000100:00000001:1.0:1787829631.869746:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000010:0.0:1787829631.869750:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc180.
00000020:00000040:1.0:1787829631.869751:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829631.869765:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829631.869767:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829631.869768:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829631.869773:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829631.869776:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829631.869778:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298452653
02000000:00000001:1.0:1787829631.869779:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829631.869786:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc960.
00000100:00000001:1.0:1787829631.869787:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829631.869793:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829631.869795:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000008:00000001:1.0:1787829631.869798:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869801:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829631.869805:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829631.869807:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829631.869811:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829631.869812:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829631.869815:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829631.869821:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829631.869823:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829631.869824:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829631.869828:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.869832:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:1.0:1787829631.869834:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829631.869839:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829631.869845:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631]  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829631.869846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829631.869895:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829631.869899:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829631.869902:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829631.869907:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:1.0:1787829631.869908:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829631.869914:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829631.869920:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829631.869926:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:1.0:1787829631.869927:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.869941:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:1.0:1787829631.869941:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3789)
00000100:00000200:0.0:1787829631.869947:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829631.869957:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.869963:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829631.869965:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.869967:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.869969:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829631.869972:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.869977:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829631.869981:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.869985:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829631.869988:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.869989:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.869990:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.869993:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829631.869994:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829631.869997:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829631.870016:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829631.870018:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829631.870022:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829631.870028:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327363456:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829631.870032:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870034:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870037:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327363456/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870041:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829631.870043:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870046:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829631.870049:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829631.870051:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829631.870052:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829631.870054:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829631.870056:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829631.870060:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829631.870061:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829631.870063:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829631.870066:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829631.870067:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870069:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870077:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829631.870078:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870080:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829631.870082:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829631.870083:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829631.870087:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870092:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829631.870094:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.870095:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829631.870097:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829631.870099:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829631.870101:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363584:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829631.870103:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829631.870105:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829631.870106:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.870108:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870110:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829631.870112:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829631.870115:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829631.870117:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829631.870120:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829631.870122:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363584, portal 10
00000100:00000001:0.0:1787829631.870124:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829631.870125:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829631.870130:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870135:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829631.870137:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829631.870138:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829631.870141:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327363584, offset 0
00000400:00000200:0.0:1787829631.870143:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829631.870146:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829631.870149:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829631.870151:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829631.870153:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829631.870155:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829631.870158:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829631.870160:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40612:40612:256:4294967295] 192.168.203.116@tcp LPNI seq info [40612:40612:8:4294967295]
00000400:00000200:0.0:1787829631.870163:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829631.870168:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829631.870172:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.870174:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829631.870177:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829631.870179:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829631.870183:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829631.870185:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829631.870188:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.870190:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829631.870192:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829631.870195:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829631.870197:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829631.870198:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829631.870200:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829631.870202:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870208:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870210:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870215:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870216:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829631.870218:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829631.870221:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829631.870223:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829631.870224:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298451979
00000100:00000001:0.0:1787829631.870226:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298452653
00000100:00000001:0.0:1787829631.870228:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298451979
00000100:00000001:0.0:1787829631.870230:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298452653
00000100:00000001:0.0:1787829631.870232:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829631.870234:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870235:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829631.870237:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631]  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829631.870243:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829631.870243:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829631.870244:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829631.870246:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829631.870247:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829631.870248:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829631.870250:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829631.870256:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870260:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829631.870261:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870262:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.870263:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829631.870266:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870273:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829631.870277:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870283:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829631.870286:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870288:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.870289:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870292:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829631.870294:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829631.870298:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829631.870302:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829631.870305:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829631.870309:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829631.870318:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363584:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829631.870323:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829631.870326:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870331:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829631.870332:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.870334:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829631.870336:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829631.870337:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829631.870339:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363712:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829631.870341:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829631.870343:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829631.870344:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.870346:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870349:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829631.870350:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829631.870353:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829631.870355:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829631.870358:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829631.870360:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363712, portal 4
00000100:00000001:0.0:1787829631.870361:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829631.870363:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829631.870367:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870376:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829631.870378:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829631.870380:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829631.870382:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327363712, offset 0
00000400:00000200:0.0:1787829631.870384:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829631.870387:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829631.870389:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829631.870391:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829631.870394:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829631.870396:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829631.870399:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829631.870402:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40613:40613:256:4294967295] 192.168.203.116@tcp LPNI seq info [40613:40613:8:4294967295]
00000400:00000200:0.0:1787829631.870407:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829631.870414:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829631.870419:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.870423:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829631.870427:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829631.870430:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829631.870435:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829631.870438:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829631.870443:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.870445:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829631.870448:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829631.870450:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829631.870452:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829631.870454:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829631.870457:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829631.870461:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870469:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870473:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870487:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870489:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829631.870493:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829631.870496:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829631.870500:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829631.870502:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298451986
00000100:00000001:0.0:1787829631.870505:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298452653
00000100:00000001:0.0:1787829631.870507:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298451986
00000100:00000001:0.0:1787829631.870509:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298452653
00000100:00000001:0.0:1787829631.870510:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829631.870512:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870513:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829631.870516:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631]  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829631.870521:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829631.870523:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829631.870524:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829631.870526:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829631.870527:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829631.870529:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829631.870531:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829631.870537:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870541:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829631.870542:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870543:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.870544:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829631.870546:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870552:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829631.870554:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870559:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829631.870560:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870561:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.870562:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870563:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829631.870564:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829631.870566:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829631.870568:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829631.870569:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829631.870571:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829631.870576:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363712:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829631.870578:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829631.870581:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870586:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829631.870587:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.870588:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829631.870590:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829631.870591:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829631.870593:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829631.870595:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829631.870597:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829631.870598:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829631.870600:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870602:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829631.870604:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829631.870606:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829631.870608:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829631.870610:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829631.870612:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363840, portal 4
00000100:00000001:0.0:1787829631.870613:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829631.870615:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:0.0:1787829631.870619:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870624:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829631.870625:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829631.870627:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829631.870628:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327363840, offset 0
00000400:00000200:0.0:1787829631.870631:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829631.870633:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829631.870635:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829631.870638:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829631.870639:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829631.870641:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829631.870643:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829631.870645:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40614:40614:256:4294967295] 192.168.203.116@tcp LPNI seq info [40614:40614:8:4294967295]
00000400:00000200:0.0:1787829631.870648:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829631.870652:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829631.870674:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.870676:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829631.870679:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829631.870681:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829631.870684:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829631.870686:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829631.870689:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829631.870691:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829631.870693:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829631.870695:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829631.870697:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829631.870699:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829631.870700:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829631.870703:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870709:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870711:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870716:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870717:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829631.870719:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829631.870721:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829631.870723:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829631.870724:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298451992
00000100:00000001:0.0:1787829631.870727:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298452653
00000100:00000001:0.0:1787829631.870728:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298451992
00000100:00000001:0.0:1787829631.870730:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298452653
00000100:00000001:0.0:1787829631.870731:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829631.870732:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870733:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829631.870736:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631]  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829631.870741:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829631.870742:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829631.870743:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829631.870745:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829631.870746:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829631.870747:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829631.870750:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829631.870755:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870759:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829631.870760:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870761:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.870762:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829631.870764:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870768:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829631.870770:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870774:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829631.870775:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870776:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829631.870777:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870778:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829631.870779:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829631.870781:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829631.870783:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829631.870785:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829631.870787:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829631.870793:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327363840:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829631.870795:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870796:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870799:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327363584/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870806:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829631.870808:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870810:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829631.870812:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829631.870814:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829631.870816:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829631.870818:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829631.870820:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829631.870823:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829631.870824:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829631.870826:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829631.870829:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829631.870831:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870833:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870837:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327363712/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870845:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829631.870846:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870848:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829631.870851:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829631.870852:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829631.870854:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829631.870856:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829631.870857:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829631.870859:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829631.870860:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829631.870861:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829631.870863:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829631.870864:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870865:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829631.870867:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327363840/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829631.870872:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829631.870873:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829631.870874:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829631.870876:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829631.870877:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829631.870877:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829631.870879:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829631.870879:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829631.870881:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829631.870881:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829631.870882:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829631.870883:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829631.870884:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870885:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829631.870888:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829631.870889:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829631.870890:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829631.870891:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829632.380755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829632.380763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829632.380765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829632.380773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829632.380778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829632.380782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829632.380784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829632.380786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829632.380790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829632.572808:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298452653, at 4298452672
00000100:00000040:3.0:1787829632.572808:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298452653, at 4298452672
00000100:00000010:2.0:1787829632.572840:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000100:00000010:3.0:1787829632.572843:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00010000:00000001:0.0:1787829632.700749:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829632.700754:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829632.700759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829632.700760:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829632.700763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829632.700764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829632.700816:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829632.700825:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829632.700828:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829633.404833:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829633.404840:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829633.404841:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829633.404846:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829633.404849:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829633.404852:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829633.404855:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829633.404873:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829633.404875:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829633.724751:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829633.724757:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829633.724761:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829633.724762:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829633.724764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829633.724765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829633.724798:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829633.724801:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829633.724804:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829634.428750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829634.428757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829634.428759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829634.428765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829634.428768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829634.428782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829634.428785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829634.428788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829634.428791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748749:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829634.748755:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829634.748757:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829634.748760:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748765:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829634.748766:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748768:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748769:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829634.748774:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829634.748775:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829634.748776:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748777:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829634.748778:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748779:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748780:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829634.748783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829634.748784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829634.748799:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829634.748801:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829634.748803:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829635.452782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829635.452788:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829635.452790:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829635.452796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829635.452799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829635.452804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829635.452806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829635.452809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829635.452812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829635.772759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829635.772764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829635.772770:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829635.772771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829635.772773:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829635.772774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829635.772824:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829635.772828:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829635.772831:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829636.476727:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829636.476732:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829636.476733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829636.476738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829636.476741:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829636.476743:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829636.476745:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829636.476747:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829636.476749:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829636.796763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829636.796770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829636.796776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829636.796777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829636.796781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829636.796782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829636.796819:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829636.796822:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829636.796825:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829636.988771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829636.988779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829636.988782:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829636.988785:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829636.988790:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829636.988795:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3784
00000400:00000200:3.0:1787829636.988802:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829636.988807:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829636.988809:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829636.988812:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988814:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829636.988816:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988817:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829636.988820:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829636.988821:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829636.988823:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829636.988826:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829636.988828:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829636.988830:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829636.988832:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988833:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829636.988835:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988836:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829636.988838:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829636.988841:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988842:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829636.988844:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829636.988847:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829636.988848:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.988850:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829636.988851:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829636.988859:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829636.988861:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829636.988865:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829636.988866:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.988868:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829636.988870:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829636.988874:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829636.988880:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829636.988884:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829636.988891:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829636.988899:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829636.988902:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829636.988905:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829636.988922:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829636.988924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829636.988926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829636.988927:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829636.988929:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829636.988931:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829636.988932:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3784
00000400:00000200:3.0:1787829636.988934:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829636.988936:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829636.988937:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829636.988939:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988941:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829636.988942:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988944:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829636.988945:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829636.988946:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829636.988947:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829636.988948:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829636.988950:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829636.988952:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829636.988954:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988956:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829636.988957:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988959:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829636.988961:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829636.988963:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829636.988975:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829636.988977:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829636.988981:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829636.988983:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.988984:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829636.988987:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00100000:3.0:1787829636.988988:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829636.988993:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.988995:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829636.988995:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829636.989000:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829636.989001:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787829636.989005:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829636.989008:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829636.989010:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989012:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829636.989013:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829636.989014:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829636.989015:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829636.989022:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829636.989027:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.989034:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00100000:3.0:1787829636.989035:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829636.989037:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.989046:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787829636.989047:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829636.989050:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.989052:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787829636.989054:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829636.989060:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829636.989070:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327363968:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829636.989075:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.989077:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787829636.989078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829636.989083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829636.989084:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:0.0:1787829636.989085:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1787829636.989085:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:0.0:1787829636.989091:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989091:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829636.989097:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3784
02000000:00000001:0.0:1787829636.989098:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:3.0:1787829636.989101:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829636.989104:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829636.989105:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000010:0.0:1787829636.989107:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
00000100:00000040:3.0:1787829636.989108:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829636.989115:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989115:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829636.989119:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.989122:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829636.989124:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829636.989127:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:3.0:1787829636.989127:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829636.989130:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829636.989133:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000010:0.0:1787829636.989135:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000020:00000001:3.0:1787829636.989135:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829636.989139:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829636.989143:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.989145:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829636.989146:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000040:3.0:1787829636.989147:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.989149:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829636.989153:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829636.989155:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000200:0.0:1787829636.989158:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327363968, portal 25
00000100:00000001:3.0:1787829636.989158:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829636.989161:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829636.989164:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1787829636.989165:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829636.989169:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
02000000:00000001:3.0:1787829636.989169:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989173:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829636.989176:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829636.989178:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989180:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829636.989183:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787829636.989184:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829636.989190:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829636.989195:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829636.989197:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989200:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829636.989204:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787829636.989204:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829636.989207:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829636.989211:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000010:3.0:1787829636.989212:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829636.989216:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829636.989217:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00100000:3.0:1787829636.989222:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829636.989225:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327363968, offset 0
00000100:00000001:3.0:1787829636.989233:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829636.989235:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829636.989240:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:3.0:1787829636.989241:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829636.989249:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829636.989251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829636.989254:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829636.989255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829636.989258:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829636.989261:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829636.989263:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829636.989266:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3784
00000400:00000200:0.0:1787829636.989268:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829636.989277:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787829636.989278:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.989283:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:3.0:1787829636.989287:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829636.989292:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1787829636.989292:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829636.989300:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:3.0:1787829636.989301:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.989306:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829636.989310:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40615:40615:256:4294967295] 192.168.203.116@tcp LPNI seq info [40615:40615:8:4294967295]
00000100:00000040:3.0:1787829636.989312:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.989319:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829636.989324:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829636.989325:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:3.0:1787829636.989328:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829636.989331:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829636.989333:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829636.989337:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829636.989341:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829636.989347:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829636.989349:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787829636.989350:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829636.989355:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829636.989360:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829636.989364:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000200:0.0:1787829636.989366:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829636.989372:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000010:0.0:1787829636.989374:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00000001:3.0:1787829636.989377:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829636.989381:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.0:1787829636.989386:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:3.0:1787829636.989387:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.0:1787829636.989393:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
02000000:00000001:3.0:1787829636.989393:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989397:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829636.989401:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.1:1787829636.989403:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:3.0:1787829636.989404:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989408:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829636.989410:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000010:3.0:1787829636.989412:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829636.989414:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829636.989416:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829636.989417:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829636.989418:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829636.989419:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829636.989421:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829636.989425:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829636.989430:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829636.989435:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00100000:3.0:1787829636.989438:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1787829636.989440:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000100:00000001:3.0:1787829636.989448:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829636.989450:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829636.989451:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787829636.989455:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829636.989463:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787829636.989465:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829636.989469:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3794)
00000400:00000200:0.0:1787829636.989472:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829636.989476:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829636.989481:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829636.989487:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989497:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.989501:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989508:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.989510:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829636.989515:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829636.989520:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829636.989523:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298457772
00000100:00000010:0.0:1787829636.989528:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc60.
00000100:00000001:0.0:1787829636.989535:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829636.989541:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298457772
00000100:00000010:0.0:1787829636.989544:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc240.
00000100:00000001:0.0:1787829636.989547:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829636.989551:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829636.989554:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.989556:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829636.989560:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636]  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829636.989569:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829636.989570:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829636.989573:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829636.989575:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829636.989577:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829636.989580:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829636.989583:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829636.989592:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989598:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829636.989600:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.989603:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.989605:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829636.989610:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989617:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829636.989621:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989629:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829636.989631:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.989633:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.989634:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.989637:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829636.989638:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829636.989641:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829636.989644:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829636.989647:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829636.989651:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829636.989695:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327363968:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829636.989700:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.989702:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.989706:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327363968/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989719:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829636.989721:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829636.989723:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829636.989727:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829636.989729:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829636.989730:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829636.989733:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829636.989735:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:0.0:1787829636.989739:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829636.989741:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829636.989743:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829636.989747:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829636.989748:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.989750:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.989757:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829636.989759:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.989761:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829636.989763:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829636.989799:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829636.989804:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989812:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829636.989814:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.989818:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829636.989820:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829636.989823:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829636.989826:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364096:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829636.989830:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829636.989833:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829636.989835:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.989837:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829636.989841:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829636.989843:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829636.989848:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829636.989851:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829636.989854:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829636.989858:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364096, portal 10
00000100:00000001:0.0:1787829636.989860:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829636.989862:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:0.0:1787829636.989868:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989874:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829636.989877:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829636.989879:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829636.989882:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327364096, offset 0
00000400:00000200:0.0:1787829636.989886:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829636.989890:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.989894:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829636.989897:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829636.989900:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829636.989902:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829636.989904:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829636.989908:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40616:40616:256:4294967295] 192.168.203.116@tcp LPNI seq info [40616:40616:8:4294967295]
00000400:00000200:0.0:1787829636.989912:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.989919:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829636.989923:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829636.989926:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829636.989931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829636.989934:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829636.989938:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829636.989941:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829636.989945:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829636.989947:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829636.989950:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829636.989953:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829636.989956:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829636.989958:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829636.989961:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829636.989980:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.989989:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.989993:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990000:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990002:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829636.990006:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829636.990009:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829636.990013:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829636.990015:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298457099
00000100:00000001:0.0:1787829636.990018:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298457772
00000100:00000001:0.0:1787829636.990021:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298457099
00000100:00000001:0.0:1787829636.990024:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298457772
00000100:00000001:0.0:1787829636.990026:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829636.990028:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990030:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829636.990035:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636]  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829636.990043:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829636.990044:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829636.990046:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829636.990048:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829636.990050:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829636.990053:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829636.990056:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829636.990064:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990077:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829636.990078:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990081:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.990082:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829636.990086:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990093:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829636.990097:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990104:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829636.990107:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990108:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.990109:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990111:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829636.990113:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829636.990116:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829636.990118:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829636.990120:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829636.990124:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829636.990132:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364096:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829636.990135:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829636.990138:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990144:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829636.990146:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.990148:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829636.990150:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829636.990152:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829636.990155:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364224:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829636.990158:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829636.990160:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829636.990161:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.990163:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829636.990167:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829636.990169:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829636.990173:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829636.990175:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829636.990178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829636.990181:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364224, portal 4
00000100:00000001:0.0:1787829636.990184:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829636.990185:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:0.0:1787829636.990190:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990197:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829636.990199:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829636.990201:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829636.990204:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327364224, offset 0
00000400:00000200:0.0:1787829636.990208:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829636.990212:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.990215:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829636.990218:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829636.990221:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829636.990224:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829636.990226:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829636.990229:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40617:40617:256:4294967295] 192.168.203.116@tcp LPNI seq info [40617:40617:8:4294967295]
00000400:00000200:0.0:1787829636.990233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.990239:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829636.990245:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829636.990248:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829636.990252:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829636.990255:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829636.990259:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829636.990262:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829636.990266:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829636.990268:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829636.990271:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829636.990276:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829636.990279:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829636.990281:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829636.990283:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829636.990288:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990294:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.990298:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990304:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990306:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829636.990308:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829636.990311:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829636.990314:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829636.990316:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298457106
00000100:00000001:0.0:1787829636.990318:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298457772
00000100:00000001:0.0:1787829636.990321:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298457106
00000100:00000001:0.0:1787829636.990323:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298457772
00000100:00000001:0.0:1787829636.990325:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829636.990327:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990329:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829636.990333:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636]  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829636.990341:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829636.990342:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829636.990344:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829636.990347:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829636.990349:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829636.990351:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829636.990354:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829636.990362:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990369:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829636.990370:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990372:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.990374:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829636.990377:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990384:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829636.990388:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990394:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829636.990397:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990398:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.990400:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990402:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829636.990403:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829636.990406:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829636.990410:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829636.990413:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829636.990417:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829636.990424:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364224:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829636.990429:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829636.990432:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990439:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829636.990441:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.990443:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829636.990444:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829636.990446:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829636.990450:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829636.990453:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829636.990455:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829636.990457:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829636.990458:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829636.990462:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829636.990464:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829636.990467:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829636.990470:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829636.990473:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829636.990476:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364352, portal 4
00000100:00000001:0.0:1787829636.990478:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829636.990479:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:0.0:1787829636.990484:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990490:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829636.990492:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829636.990495:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829636.990497:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327364352, offset 0
00000400:00000200:0.0:1787829636.990502:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829636.990505:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.990509:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829636.990512:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829636.990514:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829636.990516:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829636.990519:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829636.990521:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40618:40618:256:4294967295] 192.168.203.116@tcp LPNI seq info [40618:40618:8:4294967295]
00000400:00000200:0.0:1787829636.990526:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829636.990533:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829636.990537:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829636.990540:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829636.990545:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829636.990548:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829636.990552:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829636.990554:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829636.990558:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829636.990560:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829636.990563:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829636.990567:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829636.990569:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829636.990571:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829636.990574:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829636.990578:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990585:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.990588:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990595:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990597:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829636.990599:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829636.990602:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829636.990606:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829636.990608:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298457111
00000100:00000001:0.0:1787829636.990610:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298457772
00000100:00000001:0.0:1787829636.990612:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298457111
00000100:00000001:0.0:1787829636.990615:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298457772
00000100:00000001:0.0:1787829636.990617:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829636.990619:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990620:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829636.990624:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636]  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829636.990632:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829636.990634:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829636.990635:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829636.990637:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829636.990639:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829636.990641:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829636.990645:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829636.990653:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990677:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829636.990678:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990680:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.990682:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829636.990685:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990692:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829636.990696:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990703:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829636.990705:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990707:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829636.990708:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990710:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829636.990712:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829636.990714:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829636.990718:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829636.990720:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829636.990724:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829636.990731:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364352:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829636.990735:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990737:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.990741:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327364096/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990747:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829636.990749:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829636.990751:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829636.990754:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829636.990755:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829636.990757:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829636.990759:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829636.990761:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:0.0:1787829636.990763:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829636.990765:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829636.990766:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829636.990769:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829636.990770:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990772:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.990775:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327364224/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990787:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829636.990788:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829636.990789:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829636.990791:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829636.990793:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829636.990794:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829636.990796:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829636.990797:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:0.0:1787829636.990800:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829636.990801:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829636.990802:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:0.0:1787829636.990805:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829636.990806:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990808:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829636.990810:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327364352/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829636.990816:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829636.990818:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829636.990819:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829636.990822:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829636.990823:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829636.990824:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829636.990826:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829636.990828:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:0.0:1787829636.990830:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829636.990831:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829636.990833:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829636.990835:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829636.990836:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990838:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829636.990841:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829636.990842:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829636.990844:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829636.990845:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829637.500799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829637.500806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829637.500808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829637.500814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829637.500818:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829637.500822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829637.500825:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829637.500827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829637.500830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829637.692727:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298457772, at 4298457792
00000100:00000040:2.0:1787829637.692741:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298457772, at 4298457792
00000100:00000010:3.0:1787829637.692742:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000100:00000010:2.0:1787829637.692752:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00010000:00000001:0.0:1787829637.820738:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829637.820743:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829637.820746:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829637.820747:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829637.820750:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829637.820751:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829637.820765:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829637.820767:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829637.820769:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829638.524716:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829638.524724:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829638.524726:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829638.524730:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829638.524733:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829638.524736:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829638.524738:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829638.524740:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829638.524742:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829638.844774:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829638.844784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829638.844792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829638.844793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829638.844797:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829638.844798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829638.844879:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829638.844884:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829638.844888:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829639.548799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829639.548807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829639.548809:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829639.548814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829639.548819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829639.548822:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829639.548824:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829639.548827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829639.548830:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829639.868791:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829639.868802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829639.868811:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829639.868814:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829639.868818:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829639.868822:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829639.868825:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829639.868829:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829639.868833:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829639.868836:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829639.868838:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829639.868841:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829639.868881:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829639.868885:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829639.868887:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829640.572775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829640.572784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829640.572787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829640.572794:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829640.572799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829640.572804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829640.572807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829640.572811:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829640.572815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829640.892769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829640.892775:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829640.892779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829640.892780:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829640.892783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829640.892784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829640.892801:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829640.892803:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829640.892805:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829641.596743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829641.596750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829641.596752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829641.596758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829641.596761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829641.596764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829641.596767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829641.596769:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829641.596772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829641.916805:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829641.916811:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829641.916816:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829641.916818:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829641.916820:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829641.916821:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829641.916882:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829641.916887:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829641.916892:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829642.108806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829642.108817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829642.108820:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829642.108824:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829642.108829:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829642.108837:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3789
00000400:00000200:1.0:1787829642.108847:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829642.108853:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829642.108856:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829642.108859:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.108862:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829642.108865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.108867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829642.108871:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829642.108873:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829642.108876:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829642.108879:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829642.108881:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829642.108883:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829642.108886:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.108887:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829642.108889:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.108891:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829642.108894:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829642.108897:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829642.108898:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829642.108901:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829642.108904:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829642.108906:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.108908:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829642.108909:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829642.108920:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829642.108923:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829642.108926:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829642.108942:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.108946:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829642.108949:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829642.108955:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829642.108965:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829642.108978:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829642.108990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829642.109002:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829642.109008:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829642.109014:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829642.109039:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829642.109043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829642.109047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829642.109049:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829642.109051:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829642.109055:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829642.109059:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3789
00000100:00000001:0.0:1787829642.109060:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1787829642.109066:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829642.109067:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829642.109070:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:1.0:1787829642.109072:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829642.109077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829642.109079:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829642.109084:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109090:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829642.109096:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1787829642.109096:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829642.109099:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.109101:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829642.109104:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:1.0:1787829642.109106:0:36:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:0.0:1787829642.109107:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829642.109110:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
02000000:00000001:0.0:1787829642.109111:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829642.109116:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:0.0:1787829642.109118:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327364480:192.168.203.116@tcp:250:kworker.0
00000020:00000040:1.0:1787829642.109122:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829642.109123:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:1.0:1787829642.109128:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:0.0:1787829642.109129:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:1.0:1787829642.109133:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:0.0:1787829642.109134:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.109140:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1787829642.109143:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000010:0.0:1787829642.109146:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
00000100:00000001:1.0:1787829642.109148:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829642.109150:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829642.109154:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109158:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829642.109159:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829642.109166:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000020:00000040:1.0:1787829642.109166:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829642.109173:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000010:0.0:1787829642.109175:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000001:1.0:1787829642.109182:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829642.109184:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1787829642.109186:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364480, portal 25
00000100:00000001:0.0:1787829642.109190:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829642.109191:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
02000000:00000001:1.0:1787829642.109192:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829642.109197:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829642.109200:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829642.109201:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829642.109206:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829642.109212:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.109216:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109219:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:1.0:1787829642.109221:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829642.109225:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000040:0.0:1787829642.109226:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000001:1.0:1787829642.109229:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829642.109231:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.109232:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1787829642.109233:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
02000000:00000001:1.0:1787829642.109234:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829642.109237:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829642.109242:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327364480, offset 0
02000000:00000010:1.0:1787829642.109243:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829642.109249:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829642.109256:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829642.109257:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109265:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829642.109267:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829642.109271:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829642.109273:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829642.109282:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829642.109283:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:1.0:1787829642.109285:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829642.109290:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829642.109292:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829642.109293:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829642.109296:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829642.109299:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:1.0:1787829642.109300:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829642.109305:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1787829642.109306:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3789
00000400:00000200:0.0:1787829642.109311:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829642.109315:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40619:40619:256:4294967295] 192.168.203.116@tcp LPNI seq info [40619:40619:8:4294967295]
00000400:00000200:0.0:1787829642.109320:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829642.109325:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829642.109330:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829642.109335:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829642.109341:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787829642.109344:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109350:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829642.109354:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829642.109357:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109358:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829642.109364:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829642.109365:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000010:0.0:1787829642.109367:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000020:00000001:1.0:1787829642.109368:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829642.109373:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829642.109377:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829642.109380:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829642.109381:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829642.109388:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1787829642.109389:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109396:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787829642.109400:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829642.109402:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829642.109406:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1787829642.109408:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000100:0.0:1787829642.109414:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000040:1.0:1787829642.109415:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000010:0.0:1787829642.109417:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000100:00080000:1.0:1787829642.109424:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829642.109427:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787829642.109428:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829642.109432:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829642.109435:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1787829642.109438:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829642.109441:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
02000000:00000001:1.0:1787829642.109442:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829642.109446:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:1.0:1787829642.109447:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829642.109452:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:1.0:1787829642.109453:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000200:0.0:1787829642.109459:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829642.109459:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.109464:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829642.109472:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829642.109475:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:1.0:1787829642.109477:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000040:0.0:1787829642.109480:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829642.109481:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829642.109484:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829642.109488:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829642.109491:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829642.109494:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109495:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109498:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:1.0:1787829642.109500:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829642.109504:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109505:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829642.109510:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829642.109511:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829642.109516:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298462892
00000100:00000001:1.0:1787829642.109522:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829642.109524:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829642.109525:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd80.
00000100:00000040:1.0:1787829642.109529:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.109534:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829642.109539:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109542:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298462892
00000100:00080000:1.0:1787829642.109543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:0.0:1787829642.109549:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc00.
00000100:00080000:1.0:1787829642.109549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829642.109552:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829642.109555:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:1.0:1787829642.109555:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829642.109561:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829642.109565:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3789
00000400:00000200:1.0:1787829642.109569:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829642.109570:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829642.109575:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829642.109575:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829642.109582:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:1.0:1787829642.109582:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00100000:0.0:1787829642.109589:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829642/real 1787829642]  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829642.109590:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829642.109597:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829642.109603:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000001:0.0:1787829642.109604:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829642.109606:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829642.109608:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829642.109611:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829642.109612:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829642.109616:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829642.109617:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829642.109621:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829642.109623:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829642.109627:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829642.109631:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:0.0:1787829642.109634:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829642.109636:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829642.109643:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829642.109644:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829642.109648:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1787829642.109648:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829642.109650:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.109654:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829642.109677:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.109683:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829642.109688:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.109694:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1787829642.109697:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829642.109698:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.109703:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829642.109704:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829642.109705:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.109709:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:1.0:1787829642.109711:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829642.109712:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829642.109719:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787829642.109719:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829642.109726:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829642.109728:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829642.109732:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:1.0:1787829642.109734:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:0.0:1787829642.109737:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829642.109739:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829642.109745:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1787829642.109750:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327364480:192.168.203.116@tcp:250:kworker.0
02000000:00000001:1.0:1787829642.109751:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829642.109756:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829642.109757:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109759:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829642.109761:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:0.0:1787829642.109763:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327364480/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1787829642.109766:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829642.109769:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829642.109773:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000008:00000001:1.0:1787829642.109773:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829642.109778:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
00000100:00000001:1.0:1787829642.109778:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.109783:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000010:1.0:1787829642.109785:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000001:0.0:1787829642.109786:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829642.109789:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000040:1.0:1787829642.109791:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829642.109792:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829642.109796:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:1.0:1787829642.109798:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829642.109801:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829642.109802:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.109803:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829642.109807:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:1.0:1787829642.109808:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829642.109810:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829642.109812:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829642.109812:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829642.109816:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829642.109817:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109823:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:1.0:1787829642.109824:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:0.0:1787829642.109826:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109830:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829642.109831:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829642.109832:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109833:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829642.109838:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829642.109844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829642.109845:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829642.109847:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.109851:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829642.109854:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829642.109858:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829642.109865:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364608:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829642.109867:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829642.109870:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829642.109873:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829642.109873:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.109876:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.109879:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829642.109881:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
00000100:00000040:1.0:1787829642.109883:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829642.109884:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829642.109889:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829642.109893:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829642.109898:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829642.109903:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364608, portal 10
00000100:00000001:0.0:1787829642.109905:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829642.109906:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.109907:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:1.0:1787829642.109916:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3799)
00000100:00000040:0.0:1787829642.109918:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.109924:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829642.109926:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829642.109927:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829642.109944:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327364608, offset 0
00000400:00000200:0.0:1787829642.109948:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829642.109952:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829642.109955:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829642.109957:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829642.109959:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829642.109961:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829642.109964:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829642.109966:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40620:40620:256:4294967295] 192.168.203.116@tcp LPNI seq info [40620:40620:8:4294967295]
00000400:00000200:0.0:1787829642.109969:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829642.109974:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829642.109977:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829642.109979:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829642.109982:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829642.109984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829642.109987:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829642.109989:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829642.109991:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829642.109993:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829642.109995:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829642.109997:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829642.110000:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829642.110001:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829642.110003:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829642.110006:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110011:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829642.110013:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110018:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110019:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829642.110021:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829642.110024:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829642.110027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829642.110028:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298462219
00000100:00000001:0.0:1787829642.110030:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298462892
00000100:00000001:0.0:1787829642.110032:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298462219
00000100:00000001:0.0:1787829642.110033:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298462892
00000100:00000001:0.0:1787829642.110035:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829642.110037:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110038:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829642.110041:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829642/real 1787829642]  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829642.110046:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829642.110046:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829642.110047:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829642.110049:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829642.110050:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829642.110051:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829642.110053:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829642.110059:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110063:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829642.110064:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110065:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.110066:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829642.110068:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110072:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829642.110074:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110078:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829642.110080:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110081:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.110081:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110083:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829642.110084:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829642.110087:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829642.110091:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829642.110094:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829642.110099:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829642.110107:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364608:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829642.110112:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829642.110116:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110123:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829642.110125:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.110127:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829642.110129:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829642.110131:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829642.110134:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364736:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829642.110138:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829642.110140:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829642.110141:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.110143:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829642.110147:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829642.110149:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829642.110153:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829642.110155:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829642.110159:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829642.110162:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364736, portal 4
00000100:00000001:0.0:1787829642.110164:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829642.110166:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829642.110172:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110184:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829642.110187:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829642.110189:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829642.110192:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327364736, offset 0
00000400:00000200:0.0:1787829642.110196:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829642.110200:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829642.110204:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829642.110207:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829642.110210:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829642.110213:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829642.110216:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829642.110220:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40621:40621:256:4294967295] 192.168.203.116@tcp LPNI seq info [40621:40621:8:4294967295]
00000400:00000200:0.0:1787829642.110225:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829642.110232:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829642.110237:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829642.110241:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829642.110245:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829642.110248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829642.110253:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829642.110256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829642.110261:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829642.110263:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829642.110266:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829642.110270:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829642.110273:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829642.110276:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829642.110278:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829642.110283:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110292:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829642.110296:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110311:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110312:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829642.110316:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829642.110319:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829642.110323:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829642.110325:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298462226
00000100:00000001:0.0:1787829642.110328:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298462892
00000100:00000001:0.0:1787829642.110331:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298462226
00000100:00000001:0.0:1787829642.110333:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298462892
00000100:00000001:0.0:1787829642.110336:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829642.110338:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110340:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829642.110345:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829642/real 1787829642]  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829642.110354:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829642.110356:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829642.110358:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829642.110361:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829642.110362:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829642.110365:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829642.110369:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829642.110379:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110386:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829642.110388:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110390:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.110391:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829642.110396:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110404:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829642.110408:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110416:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829642.110419:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110420:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.110422:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110424:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829642.110426:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829642.110429:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829642.110432:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829642.110435:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829642.110439:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829642.110448:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364736:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829642.110452:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110455:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829642.110458:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327364608/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110466:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829642.110467:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829642.110469:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829642.110471:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829642.110472:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829642.110474:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829642.110475:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829642.110477:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829642.110479:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829642.110480:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829642.110481:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829642.110483:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829642.110484:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110485:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829642.110488:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327364736/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110492:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829642.110493:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829642.110494:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829642.110495:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829642.110496:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829642.110497:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829642.110498:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829642.110499:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829642.110500:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829642.110501:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829642.110501:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829642.110503:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829642.110504:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110505:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110508:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829642.110509:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110510:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829642.110511:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829642.110512:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829642.110514:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110518:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829642.110519:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.110521:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829642.110522:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829642.110524:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829642.110526:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829642.110530:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829642.110531:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829642.110532:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829642.110533:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829642.110536:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829642.110537:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829642.110539:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829642.110541:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829642.110543:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829642.110546:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364864, portal 4
00000100:00000001:0.0:1787829642.110547:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829642.110548:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:0.0:1787829642.110552:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110557:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829642.110558:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829642.110560:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829642.110561:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327364864, offset 0
00000400:00000200:0.0:1787829642.110564:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829642.110566:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829642.110568:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829642.110570:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829642.110572:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829642.110574:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829642.110576:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829642.110578:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40622:40622:256:4294967295] 192.168.203.116@tcp LPNI seq info [40622:40622:8:4294967295]
00000400:00000200:0.0:1787829642.110581:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829642.110586:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829642.110589:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829642.110592:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829642.110594:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829642.110597:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829642.110600:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829642.110601:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829642.110604:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829642.110606:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829642.110608:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829642.110610:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829642.110612:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829642.110613:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829642.110615:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829642.110617:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110623:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829642.110625:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110629:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110630:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829642.110632:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829642.110633:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829642.110635:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829642.110637:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298462231
00000100:00000001:0.0:1787829642.110638:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298462892
00000100:00000001:0.0:1787829642.110640:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298462231
00000100:00000001:0.0:1787829642.110642:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298462892
00000100:00000001:0.0:1787829642.110643:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829642.110644:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110645:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829642.110647:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829642/real 1787829642]  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829642.110652:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829642.110653:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829642.110669:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829642.110672:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829642.110673:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829642.110675:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829642.110677:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829642.110683:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110687:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829642.110688:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110689:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.110690:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829642.110692:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110697:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829642.110699:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110704:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829642.110705:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110706:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829642.110707:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110708:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829642.110709:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829642.110711:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829642.110712:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829642.110714:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829642.110716:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829642.110720:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327364864:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829642.110723:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110725:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829642.110727:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327364864/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829642.110731:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829642.110732:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829642.110733:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829642.110734:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829642.110735:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829642.110736:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829642.110737:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829642.110738:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829642.110739:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829642.110740:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829642.110742:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829642.110743:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829642.110744:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110746:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829642.110748:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829642.110749:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829642.110750:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829642.110751:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829642.620852:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829642.620862:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829642.620864:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829642.620871:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829642.620876:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829642.620881:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829642.620884:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829642.620888:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829642.620891:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829642.812730:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298462892, at 4298462912
00000100:00000040:2.0:1787829642.812733:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298462892, at 4298462912
00000100:00000010:3.0:1787829642.812748:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000100:00000010:2.0:1787829642.812750:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00010000:00000001:0.0:1787829642.940739:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829642.940746:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829642.940752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829642.940754:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829642.940757:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829642.940758:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829642.940838:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829642.940843:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829642.940846:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829643.644762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829643.644770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829643.644772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829643.644776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829643.644780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829643.644783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829643.644786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829643.644789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829643.644791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829643.964775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829643.964782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829643.964788:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829643.964790:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829643.964792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829643.964793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829643.964838:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829643.964840:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829643.964844:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829644.668768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829644.668789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829644.668792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829644.668800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829644.668804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829644.668808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829644.668812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829644.668816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829644.668820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988754:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829644.988766:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829644.988769:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829644.988773:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988779:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829644.988782:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988785:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988787:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829644.988794:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829644.988795:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829644.988813:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988815:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829644.988816:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988818:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988820:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829644.988823:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829644.988824:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829644.988887:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829644.988893:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829644.988898:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829645.692780:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829645.692797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829645.692799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829645.692806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829645.692809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829645.692813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829645.692816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829645.692820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829645.692822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829646.012770:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829646.012778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829646.012784:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829646.012786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829646.012790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829646.012791:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829646.012865:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829646.012871:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829646.012875:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829646.716831:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829646.716839:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829646.716841:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829646.716845:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829646.716849:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829646.716851:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829646.716854:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829646.716856:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829646.716858:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829647.036758:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829647.036766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829647.036771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829647.036773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829647.036777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829647.036778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829647.036830:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829647.036833:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829647.036837:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829647.228796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829647.228808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829647.228811:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829647.228816:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829647.228822:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829647.228830:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3794
00000400:00000200:1.0:1787829647.228840:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829647.228848:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829647.228853:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829647.228857:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829647.228860:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829647.228864:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829647.228868:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829647.228872:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829647.228875:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829647.228878:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829647.228882:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829647.228885:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829647.228889:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829647.228907:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829647.228910:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829647.228913:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829647.228915:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829647.228919:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829647.228922:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829647.228925:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829647.228928:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829647.228932:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829647.228935:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829647.228938:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829647.228940:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829647.228956:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829647.228960:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829647.228964:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829647.228965:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829647.228968:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829647.228971:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829647.228975:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829647.228982:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829647.228989:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829647.228998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829647.229009:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829647.229013:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829647.229018:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829647.229042:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829647.229045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829647.229054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829647.229055:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829647.229057:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829647.229061:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829647.229063:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3794
00000400:00000200:1.0:1787829647.229068:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829647.229072:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829647.229075:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829647.229079:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829647.229081:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829647.229084:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829647.229086:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829647.229087:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829647.229093:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829647.229094:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1787829647.229095:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829647.229098:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787829647.229100:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829647.229104:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000040:0.0:1787829647.229109:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829647.229109:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829647.229115:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829647.229125:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829647.229129:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829647.229131:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829647.229134:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829647.229137:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829647.229141:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787829647.229142:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829647.229145:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829647.229149:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829647.229150:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829647.229156:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:0.0:1787829647.229159:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327364992:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829647.229161:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229167:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829647.229168:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829647.229172:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829647.229175:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829647.229176:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.229180:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787829647.229182:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829647.229187:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
00000100:00000001:1.0:1787829647.229188:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829647.229193:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787829647.229195:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829647.229200:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829647.229201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829647.229205:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829647.229208:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000010:1.0:1787829647.229212:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829647.229217:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:0.0:1787829647.229218:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
02000000:00000001:1.0:1787829647.229222:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829647.229226:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829647.229228:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327364992, portal 25
00000100:00000001:1.0:1787829647.229229:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829647.229232:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829647.229234:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1787829647.229236:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229237:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
02000000:00000010:1.0:1787829647.229245:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000040:0.0:1787829647.229247:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829647.229252:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229259:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829647.229264:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00100000:1.0:1787829647.229266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829647.229268:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829647.229274:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327364992, offset 0
00000400:00000200:0.0:1787829647.229284:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829647.229284:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829647.229288:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829647.229294:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829647.229302:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.229311:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829647.229312:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829647.229316:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1787829647.229317:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829647.229322:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:1.0:1787829647.229325:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829647.229327:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787829647.229329:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829647.229331:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:1.0:1787829647.229334:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829647.229335:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40623:40623:256:4294967295] 192.168.203.116@tcp LPNI seq info [40623:40623:8:4294967295]
00000100:00000001:1.0:1787829647.229343:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829647.229344:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829647.229351:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3794
00000400:00000200:0.0:1787829647.229358:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:1.0:1787829647.229362:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.0:1787829647.229367:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829647.229372:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:0.0:1787829647.229373:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00080000:1.0:1787829647.229378:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:0.0:1787829647.229384:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829647.229388:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829647.229390:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000001:1.0:1787829647.229395:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787829647.229399:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829647.229403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000040:1.0:1787829647.229404:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000100:0.0:1787829647.229411:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829647.229411:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000010:0.0:1787829647.229413:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000020:00000001:1.0:1787829647.229419:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829647.229421:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1787829647.229423:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:0.0:1787829647.229428:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000001:1.0:1787829647.229428:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829647.229433:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829647.229435:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000020:00000001:1.0:1787829647.229439:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829647.229442:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:0.0:1787829647.229444:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000040:1.0:1787829647.229449:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829647.229450:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787829647.229455:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:0.0:1787829647.229456:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829647.229460:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829647.229467:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829647.229470:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:1.0:1787829647.229472:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000040:0.0:1787829647.229475:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829647.229480:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829647.229486:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229487:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229492:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:1.0:1787829647.229492:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829647.229498:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829647.229499:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829647.229504:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829647.229506:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829647.229510:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298468012
00000100:00000001:1.0:1787829647.229510:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829647.229515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000010:0.0:1787829647.229516:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc3c0.
00000008:00000001:1.0:1787829647.229522:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229525:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829647.229526:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829647.229533:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829647.229535:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829647.229538:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829647.229540:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298468012
02000000:00000001:1.0:1787829647.229541:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829647.229547:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:0.0:1787829647.229548:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd80.
02000000:00000001:1.0:1787829647.229549:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829647.229553:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:1.0:1787829647.229554:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229562:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:1.0:1787829647.229562:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829647.229567:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229568:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229574:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829647.229577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829647.229583:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829647/real 1787829647]  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829647.229598:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829647.229603:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829647.229606:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829647.229609:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:1.0:1787829647.229612:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829647.229613:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829647.229619:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829647.229622:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829647.229626:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829647.229630:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829647.229633:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829647.229636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829647.229663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000200:0.0:1787829647.229667:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829647.229668:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829647.229697:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829647.229699:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.229701:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.229702:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829647.229708:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829647.229708:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829647.229716:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829647.229722:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3794
00000100:00000001:0.0:1787829647.229728:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1787829647.229729:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829647.229738:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829647.229739:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829647.229746:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829647.229752:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1787829647.229755:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829647.229757:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829647.229762:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829647.229763:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.229768:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.229771:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:1.0:1787829647.229772:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829647.229773:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829647.229779:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787829647.229779:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00080000:0.0:1787829647.229787:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000020:00000001:1.0:1787829647.229787:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829647.229790:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829647.229792:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000001:1.0:1787829647.229797:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1787829647.229798:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829647.229803:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00100000:0.0:1787829647.229812:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327364992:192.168.203.116@tcp:250:kworker.0
00000020:00000001:1.0:1787829647.229812:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829647.229820:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829647.229821:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.229825:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829647.229829:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829647.229830:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327364992/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829647.229837:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829647.229842:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000040:1.0:1787829647.229845:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829647.229846:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829647.229850:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
00000100:00000001:1.0:1787829647.229852:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829647.229856:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:1.0:1787829647.229858:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000020:00000001:0.0:1787829647.229861:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000100:00080000:1.0:1787829647.229865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000020:00000040:0.0:1787829647.229866:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000100:00000001:1.0:1787829647.229872:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829647.229873:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829647.229908:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829647.229909:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829647.229916:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829647.229917:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829647.229920:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
02000000:00000001:1.0:1787829647.229922:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829647.229924:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:1.0:1787829647.229927:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829647.229930:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000008:00100000:1.0:1787829647.229931:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829647.229934:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:1.0:1787829647.229937:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229939:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829647.229942:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229949:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000010:1.0:1787829647.229949:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829647.229954:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787829647.229955:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829647.229959:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:1.0:1787829647.229961:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829647.229963:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:1.0:1787829647.229966:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.229967:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829647.229972:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829647.229975:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829647.229976:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829647.229979:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829647.229987:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:0.0:1787829647.229990:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829647.229993:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829647.229995:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.229997:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829647.230001:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829647.230005:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829647.230006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829647.230013:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365120:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829647.230023:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829647.230025:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829647.230027:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829647.230031:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.230032:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.230036:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829647.230040:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
00000100:00000040:1.0:1787829647.230041:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829647.230042:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829647.230050:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829647.230053:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829647.230059:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829647.230063:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365120, portal 10
00000100:00000001:1.0:1787829647.230064:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230065:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829647.230068:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:1.0:1787829647.230075:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3805)
00000100:00000040:0.0:1787829647.230078:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230087:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829647.230090:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829647.230094:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829647.230097:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327365120, offset 0
00000400:00000200:0.0:1787829647.230103:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829647.230107:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.230112:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829647.230115:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829647.230117:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829647.230120:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829647.230123:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829647.230126:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40624:40624:256:4294967295] 192.168.203.116@tcp LPNI seq info [40624:40624:8:4294967295]
00000400:00000200:0.0:1787829647.230130:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.230137:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829647.230141:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829647.230144:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829647.230148:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829647.230151:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829647.230156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829647.230158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829647.230162:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829647.230164:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829647.230167:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829647.230170:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829647.230173:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829647.230175:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829647.230177:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829647.230181:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230189:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829647.230192:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230199:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230200:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829647.230204:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829647.230207:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829647.230211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829647.230213:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298467339
00000100:00000001:0.0:1787829647.230215:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298468012
00000100:00000001:0.0:1787829647.230218:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298467339
00000100:00000001:0.0:1787829647.230222:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298468012
00000100:00000001:0.0:1787829647.230224:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829647.230226:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230228:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829647.230232:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829647/real 1787829647]  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829647.230239:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829647.230240:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829647.230242:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829647.230245:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829647.230246:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829647.230248:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829647.230252:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829647.230259:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230266:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829647.230267:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230269:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.230271:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829647.230274:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230280:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829647.230284:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230290:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829647.230293:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230294:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.230295:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230298:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829647.230300:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829647.230302:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829647.230306:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829647.230308:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829647.230312:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829647.230320:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365120:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829647.230323:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829647.230327:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230333:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829647.230335:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.230337:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829647.230339:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829647.230341:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829647.230344:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365248:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829647.230346:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829647.230348:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829647.230350:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.230352:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829647.230355:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829647.230357:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829647.230361:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829647.230364:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829647.230367:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829647.230370:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365248, portal 4
00000100:00000001:0.0:1787829647.230372:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829647.230374:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829647.230379:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230385:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829647.230388:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829647.230390:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829647.230392:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327365248, offset 0
00000400:00000200:0.0:1787829647.230397:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829647.230400:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.230404:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829647.230407:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829647.230410:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829647.230412:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829647.230415:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829647.230418:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40625:40625:256:4294967295] 192.168.203.116@tcp LPNI seq info [40625:40625:8:4294967295]
00000400:00000200:0.0:1787829647.230422:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.230428:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829647.230432:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829647.230435:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829647.230439:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829647.230442:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829647.230446:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829647.230448:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829647.230452:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829647.230454:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829647.230457:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829647.230460:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829647.230463:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829647.230465:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829647.230468:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829647.230471:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230478:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829647.230481:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230493:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230495:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829647.230498:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829647.230500:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829647.230504:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829647.230506:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298467346
00000100:00000001:0.0:1787829647.230508:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298468012
00000100:00000001:0.0:1787829647.230510:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298467346
00000100:00000001:0.0:1787829647.230512:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298468012
00000100:00000001:0.0:1787829647.230515:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829647.230517:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230519:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829647.230522:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829647/real 1787829647]  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829647.230529:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829647.230531:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829647.230532:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829647.230535:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829647.230536:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829647.230539:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829647.230542:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829647.230550:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230555:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829647.230557:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230559:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.230560:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829647.230564:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230571:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829647.230575:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230581:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829647.230584:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230585:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.230586:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230588:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829647.230589:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829647.230592:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829647.230595:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829647.230597:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829647.230601:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829647.230609:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365248:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829647.230613:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230614:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829647.230617:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327365120/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230624:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829647.230626:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829647.230627:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829647.230630:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829647.230632:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829647.230633:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829647.230635:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829647.230637:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829647.230640:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829647.230641:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829647.230643:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829647.230645:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829647.230647:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230648:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829647.230651:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327365248/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230677:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829647.230679:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829647.230680:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829647.230683:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829647.230684:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829647.230686:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829647.230688:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829647.230689:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829647.230691:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829647.230693:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829647.230694:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829647.230697:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829647.230698:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230700:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230703:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829647.230705:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230707:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829647.230709:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829647.230710:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829647.230714:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230721:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829647.230723:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.230725:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829647.230727:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829647.230728:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829647.230730:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829647.230735:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829647.230736:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829647.230737:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829647.230738:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829647.230741:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829647.230742:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829647.230745:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829647.230746:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829647.230764:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829647.230767:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365376, portal 4
00000100:00000001:0.0:1787829647.230768:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829647.230769:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:0.0:1787829647.230774:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230779:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829647.230780:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829647.230782:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829647.230784:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327365376, offset 0
00000400:00000200:0.0:1787829647.230787:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829647.230789:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.230792:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829647.230794:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829647.230796:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829647.230798:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829647.230800:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829647.230802:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40626:40626:256:4294967295] 192.168.203.116@tcp LPNI seq info [40626:40626:8:4294967295]
00000400:00000200:0.0:1787829647.230805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829647.230809:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829647.230813:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829647.230815:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829647.230818:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829647.230820:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829647.230823:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829647.230825:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829647.230828:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829647.230830:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829647.230832:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829647.230834:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829647.230837:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829647.230838:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829647.230840:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829647.230842:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230848:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829647.230851:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230856:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230858:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829647.230861:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829647.230864:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829647.230866:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829647.230867:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298467352
00000100:00000001:0.0:1787829647.230869:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298468012
00000100:00000001:0.0:1787829647.230871:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298467352
00000100:00000001:0.0:1787829647.230873:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298468012
00000100:00000001:0.0:1787829647.230875:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829647.230876:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.230877:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829647.230880:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829647/real 1787829647]  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829647.230885:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829647.230886:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829647.230888:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829647.230890:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829647.230891:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829647.230907:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829647.230911:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829647.230918:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230922:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829647.230923:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230925:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.230926:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829647.230929:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230934:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829647.230954:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230959:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829647.230961:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230962:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829647.230962:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230964:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829647.230966:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829647.230968:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829647.230971:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829647.230972:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829647.230974:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829647.230979:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365376:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829647.230982:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.230984:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829647.230985:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327365376/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829647.230989:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829647.230990:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829647.230991:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829647.230993:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829647.230994:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829647.230994:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829647.230995:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829647.230997:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829647.230998:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829647.230999:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829647.231000:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829647.231001:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829647.231002:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.231003:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.231006:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829647.231007:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.231008:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829647.231009:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829647.740771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829647.740780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829647.740783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829647.740788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829647.740792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829647.740796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.740799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829647.740803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829647.740806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829647.932725:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298468012, at 4298468032
00000100:00000040:3.0:1787829647.932725:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298468012, at 4298468032
00000100:00000010:3.0:1787829647.932738:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000100:00000010:2.0:1787829647.932740:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00010000:00000001:0.0:1787829648.060784:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829648.060792:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829648.060799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829648.060800:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829648.060804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829648.060806:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829648.060853:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829648.060857:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829648.060860:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829648.764765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829648.764771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829648.764772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829648.764776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829648.764778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829648.764781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829648.764783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829648.764784:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829648.764787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829649.084742:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829649.084747:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829649.084751:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829649.084752:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829649.084755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829649.084756:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829649.084805:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829649.084810:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829649.084814:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829649.788745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829649.788751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829649.788753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829649.788756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829649.788759:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829649.788761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829649.788763:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829649.788765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829649.788767:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829650.108781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829650.108786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829650.108822:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829650.108824:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829650.108829:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829650.108834:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829650.108837:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829650.108842:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829650.108846:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829650.108849:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829650.108852:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829650.108855:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829650.108951:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829650.108954:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829650.108956:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829650.812736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829650.812743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829650.812745:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829650.812748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829650.812750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829650.812754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829650.812755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829650.812757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829650.812759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829651.132766:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829651.132775:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829651.132780:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829651.132782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829651.132785:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829651.132802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829651.132846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829651.132850:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829651.132853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829651.836732:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829651.836738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829651.836740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829651.836744:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829651.836746:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829651.836749:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829651.836751:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829651.836752:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829651.836754:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829652.156793:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829652.156798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829652.156803:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829652.156804:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829652.156807:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829652.156809:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829652.156891:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829652.156895:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829652.156899:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829652.348765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829652.348775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829652.348778:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829652.348782:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829652.348787:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829652.348793:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3800
00000400:00000200:2.0:1787829652.348802:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829652.348808:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829652.348811:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829652.348815:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.348818:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829652.348820:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.348823:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829652.348827:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829652.348830:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829652.348833:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829652.348837:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829652.348839:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829652.348842:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829652.348845:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.348847:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829652.348849:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.348851:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829652.348854:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829652.348871:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829652.348874:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829652.348877:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829652.348881:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829652.348884:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.348886:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829652.348888:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829652.348898:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829652.348901:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829652.348905:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829652.348907:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.348910:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829652.348913:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829652.348917:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829652.348924:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829652.348930:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829652.348940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829652.348950:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829652.348955:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829652.348960:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829652.348987:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829652.348991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829652.348994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829652.348996:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829652.348998:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829652.349001:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829652.349004:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3800
00000100:00000001:0.0:1787829652.349006:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:2.0:1787829652.349009:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829652.349013:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829652.349016:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829652.349017:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829652.349020:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829652.349022:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349026:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829652.349028:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829652.349031:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349036:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829652.349041:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829652.349044:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829652.349046:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1787829652.349047:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829652.349049:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787829652.349051:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
02000000:00000001:0.0:1787829652.349053:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1787829652.349055:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:0.0:1787829652.349057:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1787829652.349060:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:0.0:1787829652.349061:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829652.349069:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00100000:0.0:1787829652.349070:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327365504:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787829652.349076:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829652.349077:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829652.349082:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1787829652.349083:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829652.349086:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349088:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829652.349089:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829652.349095:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
00000020:00000040:2.0:1787829652.349095:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829652.349099:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829652.349101:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829652.349106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829652.349106:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829652.349111:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
02000000:00000001:2.0:1787829652.349112:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1787829652.349118:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
02000000:00000001:2.0:1787829652.349119:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829652.349123:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829652.349125:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365504, portal 25
00000100:00000001:2.0:1787829652.349126:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829652.349130:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829652.349132:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000008:00100000:2.0:1787829652.349132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829652.349139:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829652.349141:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829652.349142:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829652.349150:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829652.349151:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1787829652.349154:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000040:0.0:1787829652.349156:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829652.349159:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1787829652.349161:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
02000000:00000001:2.0:1787829652.349163:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829652.349165:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327365504, offset 0
00000100:00000001:2.0:1787829652.349167:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829652.349170:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829652.349173:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829652.349174:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349181:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:2.0:1787829652.349181:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829652.349188:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349191:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829652.349197:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:2.0:1787829652.349198:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829652.349202:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787829652.349211:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829652.349214:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829652.349215:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349218:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829652.349225:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40627:40627:256:4294967295] 192.168.203.116@tcp LPNI seq info [40627:40627:8:4294967295]
00000100:00000040:2.0:1787829652.349225:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829652.349235:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829652.349245:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349249:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1787829652.349250:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829652.349257:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:0.0:1787829652.349260:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349260:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829652.349262:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829652.349265:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:0.0:1787829652.349267:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00080000:2.0:1787829652.349268:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3800
00000400:00000200:2.0:1787829652.349274:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.0:1787829652.349277:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:2.0:1787829652.349278:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:0.0:1787829652.349283:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00080000:2.0:1787829652.349283:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829652.349288:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349292:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787829652.349293:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829652.349297:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829652.349298:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787829652.349300:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829652.349302:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829652.349302:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829652.349303:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829652.349305:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829652.349306:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829652.349307:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829652.349309:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349311:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829652.349314:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349318:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829652.349320:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000100:0.0:1787829652.349324:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829652.349324:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349328:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829652.349330:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
02000000:00000001:2.0:1787829652.349330:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829652.349334:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829652.349337:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349338:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829652.349339:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:2.0:1787829652.349340:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829652.349345:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349346:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349351:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000010:2.0:1787829652.349352:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000400:00000200:0.0:1787829652.349360:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829652.349362:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000020:00000040:2.0:1787829652.349364:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:0.0:1787829652.349366:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787829652.349368:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829652.349370:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349373:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829652.349374:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1787829652.349375:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829652.349376:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829652.349381:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829652.349386:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829652.349394:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829652.349397:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829652.349401:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829652.349404:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829652.349406:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829652.349410:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829652.349436:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349438:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829652.349439:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829652.349442:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:2.0:1787829652.349444:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829652.349446:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829652.349447:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829652.349449:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349450:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829652.349454:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3800
00000400:00000200:0.0:1787829652.349457:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829652.349458:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829652.349462:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298473132
00000400:00000200:2.0:1787829652.349463:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829652.349466:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829652.349468:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349469:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829652.349470:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349472:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829652.349476:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829652.349476:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000010:0.0:1787829652.349478:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc00.
00000020:00000001:2.0:1787829652.349478:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829652.349480:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829652.349483:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829652.349485:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829652.349487:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829652.349488:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787829652.349489:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829652.349494:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829652.349496:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298473132
00000100:00000001:2.0:1787829652.349496:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000010:0.0:1787829652.349502:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc60.
00000020:00000040:2.0:1787829652.349502:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829652.349507:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829652.349510:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349511:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:2.0:1787829652.349511:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829652.349515:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829652.349517:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349518:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829652.349520:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829652.349522:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1787829652.349523:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349528:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349529:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349532:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:2.0:1787829652.349535:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000100:00100000:0.0:1787829652.349539:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829652/real 1787829652]  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829652.349541:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829652.349546:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829652.349549:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829652.349553:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829652.349555:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829652.349559:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829652.349563:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829652.349566:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000010:2.0:1787829652.349566:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000400:00000010:0.0:1787829652.349569:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000001:2.0:1787829652.349573:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349574:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829652.349576:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829652.349578:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829652.349581:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829652.349590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829652.349591:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349619:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829652.349619:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829652.349622:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829652.349625:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349626:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.349628:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829652.349638:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829652.349638:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349651:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829652.349684:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829652.349685:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829652.349690:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3810)
00000100:00000001:0.0:1787829652.349700:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829652.349704:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349706:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.349708:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349710:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829652.349711:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829652.349714:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829652.349717:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829652.349719:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829652.349724:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829652.349730:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327365504:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829652.349734:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349736:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.349737:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327365504/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349742:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829652.349744:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829652.349746:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829652.349749:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829652.349751:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829652.349752:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829652.349753:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829652.349755:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829652.349758:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829652.349760:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829652.349761:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:0.0:1787829652.349763:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829652.349764:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349766:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349772:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829652.349773:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349775:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829652.349776:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829652.349777:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829652.349779:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349784:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829652.349785:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829652.349787:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829652.349788:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829652.349790:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829652.349792:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365632:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829652.349795:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829652.349796:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829652.349797:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829652.349798:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829652.349801:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829652.349802:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.349805:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829652.349807:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829652.349809:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829652.349811:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365632, portal 10
00000100:00000001:0.0:1787829652.349813:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829652.349814:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:0.0:1787829652.349819:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349823:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829652.349824:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829652.349826:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829652.349828:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327365632, offset 0
00000400:00000200:0.0:1787829652.349830:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829652.349833:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829652.349836:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829652.349838:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829652.349839:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829652.349841:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829652.349843:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829652.349845:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40628:40628:256:4294967295] 192.168.203.116@tcp LPNI seq info [40628:40628:8:4294967295]
00000400:00000200:0.0:1787829652.349848:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829652.349853:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829652.349855:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829652.349872:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829652.349875:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829652.349878:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829652.349881:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829652.349883:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829652.349885:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829652.349887:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829652.349889:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829652.349892:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829652.349894:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829652.349895:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829652.349897:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829652.349900:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349905:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.349907:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349911:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349912:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829652.349915:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829652.349917:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829652.349920:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829652.349921:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298472459
00000100:00000001:0.0:1787829652.349923:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298473132
00000100:00000001:0.0:1787829652.349925:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298472459
00000100:00000001:0.0:1787829652.349926:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298473132
00000100:00000001:0.0:1787829652.349928:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829652.349929:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.349930:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829652.349932:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829652/real 1787829652]  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829652.349938:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829652.349939:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829652.349940:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829652.349942:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829652.349943:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829652.349945:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829652.349947:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829652.349952:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349956:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829652.349957:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349958:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.349959:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829652.349961:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349966:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829652.349968:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349972:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829652.349974:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349975:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.349975:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.349977:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829652.349978:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829652.349979:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829652.349981:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829652.349983:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829652.349985:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829652.349990:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365632:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829652.349992:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829652.349995:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.349999:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829652.350000:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829652.350001:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829652.350003:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829652.350004:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829652.350006:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829652.350007:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829652.350009:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829652.350010:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829652.350011:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829652.350013:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829652.350014:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.350017:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829652.350018:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829652.350020:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829652.350022:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365760, portal 4
00000100:00000001:0.0:1787829652.350023:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829652.350024:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:0.0:1787829652.350028:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350032:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829652.350034:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829652.350035:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829652.350037:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327365760, offset 0
00000400:00000200:0.0:1787829652.350039:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829652.350042:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829652.350044:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829652.350046:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829652.350048:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829652.350050:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829652.350052:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829652.350054:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40629:40629:256:4294967295] 192.168.203.116@tcp LPNI seq info [40629:40629:8:4294967295]
00000400:00000200:0.0:1787829652.350056:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829652.350060:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829652.350063:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829652.350065:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829652.350067:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829652.350069:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829652.350071:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829652.350073:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829652.350076:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829652.350077:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829652.350079:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829652.350081:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829652.350082:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829652.350084:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829652.350085:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829652.350088:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350093:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.350094:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350099:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.350100:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829652.350102:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829652.350103:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829652.350106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829652.350107:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298472466
00000100:00000001:0.0:1787829652.350109:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298473132
00000100:00000001:0.0:1787829652.350110:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298472466
00000100:00000001:0.0:1787829652.350112:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298473132
00000100:00000001:0.0:1787829652.350113:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829652.350115:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.350116:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829652.350118:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829652/real 1787829652]  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829652.350122:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829652.350123:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829652.350124:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829652.350125:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829652.350126:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829652.350128:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829652.350129:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829652.350135:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350139:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829652.350140:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350141:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.350142:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829652.350144:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350148:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829652.350150:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350154:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829652.350155:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350156:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.350157:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350158:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829652.350159:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829652.350161:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829652.350163:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829652.350165:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829652.350167:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829652.350172:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829652.350174:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829652.350176:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350179:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829652.350180:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829652.350182:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829652.350183:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829652.350184:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829652.350185:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365888:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829652.350187:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829652.350188:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829652.350189:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829652.350190:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829652.350192:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829652.350193:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829652.350195:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829652.350196:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829652.350198:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829652.350200:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327365888, portal 4
00000100:00000001:0.0:1787829652.350201:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829652.350202:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:0.0:1787829652.350205:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350209:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829652.350211:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829652.350212:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829652.350214:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327365888, offset 0
00000400:00000200:0.0:1787829652.350216:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829652.350218:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829652.350220:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829652.350221:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829652.350223:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829652.350224:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829652.350226:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829652.350228:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40630:40630:256:4294967295] 192.168.203.116@tcp LPNI seq info [40630:40630:8:4294967295]
00000400:00000200:0.0:1787829652.350231:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829652.350235:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829652.350237:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829652.350239:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829652.350241:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829652.350242:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829652.350245:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829652.350247:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829652.350249:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829652.350250:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829652.350252:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829652.350254:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829652.350256:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829652.350257:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829652.350258:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829652.350260:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350264:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.350267:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350275:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.350276:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829652.350277:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829652.350279:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829652.350281:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829652.350282:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298472471
00000100:00000001:0.0:1787829652.350284:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298473132
00000100:00000001:0.0:1787829652.350285:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298472471
00000100:00000001:0.0:1787829652.350286:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298473132
00000100:00000001:0.0:1787829652.350288:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829652.350289:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.350290:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829652.350292:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829652/real 1787829652]  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829652.350296:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829652.350297:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829652.350298:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829652.350300:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829652.350302:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829652.350304:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829652.350307:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829652.350316:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350323:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829652.350324:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350326:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.350328:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829652.350332:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350340:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829652.350345:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350352:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829652.350354:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350355:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829652.350356:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350357:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829652.350359:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829652.350361:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829652.350363:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829652.350364:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829652.350368:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829652.350376:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327365888:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829652.350380:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350382:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.350386:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327365632/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350393:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829652.350395:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829652.350397:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829652.350400:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829652.350401:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829652.350403:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829652.350405:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829652.350407:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829652.350410:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829652.350411:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829652.350413:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829652.350415:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829652.350416:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350417:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.350420:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327365760/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350424:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829652.350425:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829652.350427:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829652.350428:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829652.350429:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829652.350430:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829652.350431:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829652.350432:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829652.350433:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829652.350434:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829652.350434:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:0.0:1787829652.350436:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829652.350437:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350437:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829652.350439:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327365888/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829652.350443:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829652.350444:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829652.350445:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829652.350446:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829652.350446:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829652.350447:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829652.350448:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829652.350449:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829652.350451:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829652.350451:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829652.350452:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:0.0:1787829652.350454:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829652.350455:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350456:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.350459:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829652.350460:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.350461:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829652.350462:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829652.860727:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829652.860732:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829652.860734:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829652.860738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829652.860741:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829652.860744:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.860746:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829652.860749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829652.860751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829653.052758:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298473132, at 4298473152
00000100:00000040:3.0:1787829653.052758:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298473132, at 4298473152
00000100:00000010:3.0:1787829653.052772:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00000100:00000010:2.0:1787829653.052774:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00010000:00000001:0.0:1787829653.180767:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829653.180773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829653.180778:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829653.180779:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829653.180782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829653.180783:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829653.180821:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829653.180824:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829653.180826:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000008:00000020:2.0:1787829653.372814:0:16289:0:(osc_request.c:747:osc_update_next_shrink()) next time 5006 to shrink grant
00000008:00000020:2.0:1787829653.372821:0:16289:0:(osc_request.c:747:osc_update_next_shrink()) next time 5006 to shrink grant
00000100:00000001:0.0:1787829653.884736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829653.884744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829653.884746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829653.884750:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829653.884754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829653.884757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829653.884759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829653.884761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829653.884763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829654.204772:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829654.204774:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829654.204776:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204781:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829654.204784:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204786:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204787:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829654.204791:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829654.204792:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829654.204793:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204794:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829654.204795:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204796:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204797:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829654.204799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829654.204800:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829654.204857:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829654.204861:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829654.204864:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829654.908786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829654.908792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829654.908793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829654.908798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829654.908801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829654.908803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829654.908805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829654.908807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829654.908810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829655.228754:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829655.228759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829655.228765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829655.228766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829655.228768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829655.228769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829655.228814:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829655.228817:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829655.228820:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829655.932751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829655.932757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829655.932758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829655.932763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829655.932766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829655.932768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829655.932770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829655.932772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829655.932774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829656.252742:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829656.252748:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829656.252752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829656.252753:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829656.252756:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829656.252757:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829656.252773:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829656.252775:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829656.252777:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829656.956773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829656.956779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829656.956781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829656.956784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829656.956787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829656.956790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829656.956792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829656.956794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829656.956796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829657.276756:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829657.276763:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829657.276768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829657.276769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829657.276772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829657.276773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829657.276841:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829657.276844:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829657.276846:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829657.468811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829657.468819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829657.468833:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829657.468836:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829657.468841:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829657.468847:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3805
00000400:00000200:1.0:1787829657.468855:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829657.468861:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829657.468864:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829657.468867:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.468870:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.468872:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.468873:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829657.468876:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829657.468878:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829657.468879:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829657.468882:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829657.468884:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829657.468886:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829657.468888:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.468889:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.468891:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.468892:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829657.468895:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829657.468897:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829657.468899:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829657.468901:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829657.468904:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829657.468906:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.468907:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829657.468909:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829657.468917:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829657.468920:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829657.468923:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829657.468924:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.468926:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829657.468928:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829657.468931:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829657.468939:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829657.468943:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829657.468951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829657.468959:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829657.468962:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829657.468966:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829657.468981:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829657.468984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829657.468986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829657.468987:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829657.468988:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829657.468990:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829657.468992:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3805
00000400:00000200:1.0:1787829657.468995:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829657.468997:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829657.468999:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829657.469001:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469003:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.469005:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469007:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829657.469009:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829657.469010:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829657.469011:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829657.469013:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829657.469015:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829657.469016:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829657.469018:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469019:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.469020:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829657.469022:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829657.469022:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829657.469027:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829657.469028:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787829657.469030:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829657.469031:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829657.469033:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829657.469035:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829657.469040:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829657.469041:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829657.469046:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.469050:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829657.469055:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829657.469056:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829657.469060:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1787829657.469061:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.469065:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469066:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:1.0:1787829657.469068:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
02000000:00000001:0.0:1787829657.469070:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829657.469071:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829657.469074:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829657.469075:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829657.469079:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829657.469081:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327366016:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829657.469082:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829657.469085:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829657.469086:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829657.469089:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:0.0:1787829657.469094:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829657.469094:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469100:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:1.0:1787829657.469101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829657.469103:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469109:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829657.469113:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829657.469115:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829657.469116:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829657.469121:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829657.469121:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829657.469131:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829657.469135:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829657.469136:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00080000:1.0:1787829657.469140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787829657.469143:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00080000:1.0:1787829657.469146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829657.469150:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:0.0:1787829657.469151:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366016, portal 25
00000100:00080000:1.0:1787829657.469152:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829657.469155:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829657.469158:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000001:1.0:1787829657.469158:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829657.469164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3805
00000100:00000040:0.0:1787829657.469167:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829657.469174:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829657.469181:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787829657.469182:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000040:0.0:1787829657.469188:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829657.469188:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000010:0.0:1787829657.469192:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000040:1.0:1787829657.469195:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829657.469198:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327366016, offset 0
00000100:00000001:1.0:1787829657.469199:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.469203:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469205:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829657.469208:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829657.469209:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:1.0:1787829657.469210:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829657.469213:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829657.469215:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829657.469217:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829657.469220:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:1.0:1787829657.469221:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829657.469227:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829657.469230:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829657.469231:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.469236:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469238:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829657.469239:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1787829657.469242:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829657.469246:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:1.0:1787829657.469247:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469251:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469254:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787829657.469254:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829657.469259:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:1.0:1787829657.469259:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829657.469264:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469269:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40631:40631:256:4294967295] 192.168.203.116@tcp LPNI seq info [40631:40631:8:4294967295]
00000100:00000001:1.0:1787829657.469269:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829657.469272:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829657.469277:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469278:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829657.469281:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829657.469286:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829657.469290:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829657.469292:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1787829657.469294:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829657.469297:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.469299:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.0:1787829657.469302:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.469307:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
02000000:00000001:1.0:1787829657.469310:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829657.469312:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829657.469316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000010:1.0:1787829657.469318:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000800:00000200:0.0:1787829657.469321:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787829657.469323:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829657.469327:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:1.0:1787829657.469330:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829657.469340:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:0.1:1787829657.469340:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787829657.469345:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829657.469349:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829657.469350:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1787829657.469355:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000100:00000001:1.0:1787829657.469361:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469363:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1787829657.469365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829657.469371:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1787829657.469371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829657.469374:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829657.469376:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000100:00080000:1.0:1787829657.469377:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:0.0:1787829657.469381:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:1.0:1787829657.469382:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829657.469385:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:1.0:1787829657.469386:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3805
00000100:00000200:0.0:1787829657.469392:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829657.469392:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829657.469398:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829657.469400:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829657.469401:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469405:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.469409:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469413:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829657.469415:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:1.0:1787829657.469415:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829657.469418:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000040:0.0:1787829657.469422:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829657.469422:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829657.469424:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829657.469428:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829657.469432:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829657.469439:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829657.469439:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829657.469443:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829657.469444:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829657.469448:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829657.469450:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829657.469451:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787829657.469452:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829657.469458:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829657.469461:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829657.469462:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469465:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298478252
02000000:00000001:1.0:1787829657.469466:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829657.469470:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1787829657.469472:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc240.
02000000:00000001:1.0:1787829657.469474:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.469478:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829657.469480:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000008:00100000:1.0:1787829657.469481:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829657.469486:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.469488:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469491:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298478252
00000100:00000010:1.0:1787829657.469493:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829657.469498:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829657.469501:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829657.469502:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829657.469504:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:0.0:1787829657.469505:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc960.
02000000:00000001:1.0:1787829657.469506:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829657.469510:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469512:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000010:1.0:1787829657.469517:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829657.469522:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469524:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787829657.469528:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829657.469530:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469537:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829657.469540:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00100000:0.0:1787829657.469545:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657]  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829657.469545:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829657.469556:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829657.469562:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829657.469565:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829657.469569:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000001:1.0:1787829657.469570:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469573:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829657.469576:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000040:1.0:1787829657.469577:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3815)
00000100:00000001:0.0:1787829657.469580:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829657.469586:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829657.469593:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469599:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829657.469601:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469603:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.469604:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829657.469607:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469611:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829657.469614:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469619:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829657.469621:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469622:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.469623:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469624:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829657.469625:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829657.469627:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829657.469630:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829657.469631:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829657.469634:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829657.469639:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327366016:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829657.469642:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469643:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.469645:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327366016/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469654:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829657.469683:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829657.469685:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829657.469688:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829657.469689:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829657.469691:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829657.469692:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829657.469694:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:0.0:1787829657.469697:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829657.469698:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829657.469699:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829657.469701:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829657.469702:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469704:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469709:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829657.469710:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469712:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829657.469713:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829657.469714:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829657.469717:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469722:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829657.469723:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469724:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829657.469726:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829657.469727:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829657.469729:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366144:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829657.469732:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829657.469733:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829657.469734:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469736:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829657.469738:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829657.469739:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469742:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829657.469744:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829657.469746:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829657.469749:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366144, portal 10
00000100:00000001:0.0:1787829657.469750:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829657.469751:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829657.469756:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469760:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829657.469762:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829657.469763:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829657.469765:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327366144, offset 0
00000400:00000200:0.0:1787829657.469767:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829657.469770:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829657.469772:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829657.469774:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829657.469776:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829657.469778:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829657.469779:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829657.469782:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40632:40632:256:4294967295] 192.168.203.116@tcp LPNI seq info [40632:40632:8:4294967295]
00000400:00000200:0.0:1787829657.469784:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829657.469789:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829657.469792:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.469794:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829657.469797:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829657.469799:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829657.469802:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829657.469804:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829657.469807:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.469808:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829657.469810:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829657.469812:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829657.469814:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829657.469816:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829657.469818:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829657.469820:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469838:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.469840:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469845:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469846:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829657.469849:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829657.469851:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829657.469853:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829657.469854:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298477579
00000100:00000001:0.0:1787829657.469857:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298478252
00000100:00000001:0.0:1787829657.469858:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298477579
00000100:00000001:0.0:1787829657.469860:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298478252
00000100:00000001:0.0:1787829657.469861:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829657.469863:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.469864:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829657.469870:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657]  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829657.469875:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829657.469876:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829657.469877:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829657.469879:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829657.469880:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829657.469882:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829657.469884:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829657.469890:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469898:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829657.469899:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469900:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.469901:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829657.469903:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469907:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829657.469909:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469914:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829657.469915:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469916:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.469917:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.469918:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829657.469919:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829657.469921:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829657.469923:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829657.469925:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829657.469927:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829657.469932:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366144:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829657.469934:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829657.469936:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469940:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829657.469941:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469943:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829657.469944:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829657.469945:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829657.469947:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829657.469950:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829657.469951:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829657.469952:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.469953:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829657.469955:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829657.469956:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.469959:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829657.469961:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829657.469963:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829657.469964:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366272, portal 4
00000100:00000001:0.0:1787829657.469966:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829657.469967:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829657.469970:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.469974:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829657.469976:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829657.469977:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829657.469979:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327366272, offset 0
00000400:00000200:0.0:1787829657.469981:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829657.469983:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829657.469985:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829657.469987:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829657.469989:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829657.469990:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829657.469992:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829657.469994:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40633:40633:256:4294967295] 192.168.203.116@tcp LPNI seq info [40633:40633:8:4294967295]
00000400:00000200:0.0:1787829657.469997:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829657.470001:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829657.470004:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.470006:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829657.470009:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829657.470011:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829657.470013:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829657.470015:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829657.470018:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.470019:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829657.470021:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829657.470023:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829657.470025:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829657.470026:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829657.470028:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829657.470030:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470035:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.470037:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470041:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.470042:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829657.470044:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829657.470046:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829657.470048:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829657.470049:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298477586
00000100:00000001:0.0:1787829657.470051:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298478252
00000100:00000001:0.0:1787829657.470052:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298477586
00000100:00000001:0.0:1787829657.470053:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298478252
00000100:00000001:0.0:1787829657.470055:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829657.470056:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.470057:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829657.470059:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657]  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829657.470065:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829657.470065:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829657.470067:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829657.470068:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829657.470069:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829657.470070:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829657.470072:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829657.470077:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470081:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829657.470082:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470084:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.470085:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829657.470087:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470091:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829657.470093:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470097:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829657.470098:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470099:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.470100:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470101:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829657.470102:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829657.470103:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829657.470105:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829657.470107:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829657.470109:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829657.470114:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829657.470117:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829657.470119:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470123:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829657.470124:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.470125:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829657.470126:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829657.470127:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829657.470129:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366400:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829657.470130:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829657.470131:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829657.470132:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829657.470133:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829657.470135:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829657.470136:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829657.470138:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829657.470139:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829657.470141:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829657.470143:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366400, portal 4
00000100:00000001:0.0:1787829657.470144:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829657.470145:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:0.0:1787829657.470148:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470152:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829657.470153:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829657.470155:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829657.470156:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327366400, offset 0
00000400:00000200:0.0:1787829657.470159:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829657.470161:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829657.470163:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829657.470165:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829657.470166:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829657.470168:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829657.470170:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829657.470171:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40634:40634:256:4294967295] 192.168.203.116@tcp LPNI seq info [40634:40634:8:4294967295]
00000400:00000200:0.0:1787829657.470174:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829657.470178:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829657.470180:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.470182:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829657.470184:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829657.470186:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829657.470189:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829657.470190:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829657.470193:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829657.470194:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829657.470196:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829657.470198:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829657.470199:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829657.470200:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829657.470201:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829657.470204:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470208:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.470210:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470214:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.470215:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829657.470217:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829657.470219:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829657.470220:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829657.470222:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298477591
00000100:00000001:0.0:1787829657.470224:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298478252
00000100:00000001:0.0:1787829657.470225:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298477591
00000100:00000001:0.0:1787829657.470226:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298478252
00000100:00000001:0.0:1787829657.470227:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829657.470229:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.470230:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829657.470232:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657]  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829657.470236:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829657.470237:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829657.470238:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829657.470239:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829657.470240:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829657.470241:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829657.470243:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829657.470248:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470252:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829657.470253:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470254:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.470255:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829657.470257:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470261:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829657.470263:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470267:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829657.470269:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470270:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829657.470271:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470272:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829657.470273:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829657.470275:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829657.470277:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829657.470278:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829657.470280:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829657.470285:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366400:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829657.470287:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470288:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.470290:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327366144/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470294:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829657.470295:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829657.470296:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829657.470297:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829657.470298:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829657.470299:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829657.470300:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829657.470301:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:0.0:1787829657.470303:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829657.470304:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829657.470305:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829657.470307:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829657.470307:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470308:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.470310:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327366272/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470318:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829657.470319:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829657.470320:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829657.470321:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829657.470322:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829657.470323:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829657.470324:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829657.470325:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829657.470326:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829657.470327:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829657.470327:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829657.470329:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829657.470329:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470330:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829657.470332:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327366400/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829657.470336:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829657.470336:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829657.470337:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829657.470338:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829657.470339:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829657.470340:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829657.470341:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829657.470341:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829657.470343:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829657.470343:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829657.470344:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829657.470346:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829657.470347:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470348:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.470349:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829657.470350:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.470351:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829657.470353:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829657.980773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829657.980779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829657.980781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829657.980785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829657.980788:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829657.980791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.980792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829657.980794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829657.980796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829658.172759:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298478252, at 4298478272
00000100:00000040:3.0:1787829658.172759:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298478252, at 4298478272
00000100:00000010:3.0:1787829658.172777:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000100:00000010:2.0:1787829658.172780:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00010000:00000001:0.0:1787829658.300753:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829658.300759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829658.300763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829658.300765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829658.300767:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829658.300768:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829658.300813:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829658.300829:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829658.300832:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829659.004756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829659.004762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829659.004764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829659.004768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829659.004770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829659.004773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829659.004775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829659.004777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829659.004779:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829659.324731:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829659.324737:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829659.324741:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829659.324742:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829659.324745:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829659.324748:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829659.324751:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829659.324754:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829659.324757:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829659.324759:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829659.324761:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829659.324764:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829659.324784:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829659.324786:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829659.324788:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829660.028782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829660.028789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829660.028791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829660.028796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829660.028799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829660.028803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829660.028817:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829660.028820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829660.028823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829660.348734:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829660.348739:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829660.348744:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829660.348745:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829660.348748:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829660.348749:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829660.348786:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829660.348790:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829660.348792:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829661.052758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829661.052765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829661.052768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829661.052772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829661.052776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829661.052779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829661.052782:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829661.052785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829661.052788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829661.372762:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829661.372769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829661.372776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829661.372778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829661.372781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829661.372782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829661.372854:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829661.372857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829661.372860:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829662.076818:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829662.076826:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829662.076828:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829662.076833:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829662.076837:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829662.076841:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829662.076843:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829662.076846:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829662.076849:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829662.396777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829662.396784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829662.396801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829662.396802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829662.396805:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829662.396806:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829662.396831:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829662.396834:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829662.396838:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829662.588760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829662.588771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829662.588773:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829662.588778:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829662.588783:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829662.588806:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3810
00000400:00000200:0.0:1787829662.588816:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829662.588823:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829662.588826:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829662.588829:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.588831:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.588833:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.588835:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829662.588838:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829662.588840:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829662.588842:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829662.588845:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829662.588847:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829662.588849:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829662.588851:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.588853:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.588854:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.588856:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829662.588859:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829662.588862:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829662.588864:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829662.588867:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829662.588870:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829662.588872:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.588874:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829662.588876:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829662.588886:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716e580.
00000020:00000040:0.0:1787829662.588889:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829662.588892:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829662.588894:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.588897:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829662.588900:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829662.588904:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829662.588912:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531c00.
00000100:00000001:0.0:1787829662.588919:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829662.588928:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829662.588937:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829662.588940:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829662.588944:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc9f2c41] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829662.588963:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829662.588966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829662.588969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829662.588970:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829662.588971:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829662.588974:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829662.588976:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3810
00000400:00000200:0.0:1787829662.588979:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829662.588982:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829662.588984:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829662.588986:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.588988:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.588990:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829662.588990:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829662.588996:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829662.588997:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829662.588998:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:1.0:1787829662.588999:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787829662.589000:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829662.589002:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829662.589004:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:0.0:1787829662.589006:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829662.589008:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589009:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.589012:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589014:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829662.589014:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787829662.589018:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:0.0:1787829662.589021:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589023:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829662.589025:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787829662.589027:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829662.589030:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829662.589035:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589055:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829662.589063:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829662.589065:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:0.0:1787829662.589067:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589071:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589072:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829662.589078:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f0c0.
00000020:00000040:0.0:1787829662.589083:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829662.589083:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829662.589089:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829662.589091:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829662.589092:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589097:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829662.589101:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829662.589101:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829662.589122:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829662.589126:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327366528:192.168.203.116@tcp:250:kworker.0
02000000:00000010:0.0:1787829662.589128:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532000.
00000100:00000001:0.0:1787829662.589133:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589139:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:0.0:1787829662.589141:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787829662.589147:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787829662.589149:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787829662.589152:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589153:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829662.589161:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003ce953a3] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829662.589161:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829662.589170:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829662.589171:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
00000100:00080000:0.0:1787829662.589173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787829662.589178:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829662.589196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829662.589200:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829662.589202:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829662.589205:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829662.589209:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829662.589212:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3810
00000400:00000200:0.0:1787829662.589218:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829662.589219:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000200:0.0:1787829662.589222:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829662.589227:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829662.589230:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829662.589232:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000001:0.0:1787829662.589233:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.589238:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589242:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829662.589244:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829662.589245:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000200:1.0:1787829662.589247:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366528, portal 25
00000020:00000001:0.0:1787829662.589248:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829662.589252:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000001:1.0:1787829662.589255:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1787829662.589256:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:1.0:1787829662.589260:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000020:00000001:0.0:1787829662.589261:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:0.0:1787829662.589270:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589289:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829662.589291:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829662.589293:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589297:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829662.589301:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:0.0:1787829662.589306:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589311:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829662.589314:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829662.589314:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787829662.589318:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829662.589323:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829662.589323:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589327:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787829662.589329:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000008:00100000:0.0:1787829662.589330:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829662.589335:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589336:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829662.589337:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327366528, offset 0
00000100:00000010:0.0:1787829662.589339:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716c000.
00000020:00000040:0.0:1787829662.589343:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:0.0:1787829662.589345:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829662.589346:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589348:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787829662.589351:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:0.0:1787829662.589352:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829662.589356:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829662.589377:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531800.
00000100:00000001:0.0:1787829662.589381:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.589382:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:0.0:1787829662.589392:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829662.589396:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829662.589407:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787829662.589409:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829662.589414:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.589415:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829662.589425:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1787829662.589426:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cdd0ad0f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829662.589431:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829662.589439:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40635:40635:256:4294967295] 192.168.203.116@tcp LPNI seq info [40635:40635:8:4294967295]
00000100:00000001:0.0:1787829662.589449:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.589452:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829662.589455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829662.589459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829662.589461:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829662.589463:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829662.589469:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787829662.589477:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1787829662.589482:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:0.0:1787829662.589485:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3810
00000800:00000010:1.0:1787829662.589507:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000400:00000200:0.0:1787829662.589509:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829662.589515:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829662.589517:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1787829662.589522:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:0.0:1787829662.589523:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589528:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.589530:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829662.589530:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787829662.589534:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829662.589537:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829662.589539:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:0.0:1787829662.589541:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.1:1787829662.589543:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:0.0:1787829662.589544:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:0.0:1787829662.589548:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829662.589551:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:1.1:1787829662.589551:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1787829662.589557:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589559:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829662.589564:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000100:1.0:1787829662.589564:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589568:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000010:1.0:1787829662.589571:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000020:00000040:0.0:1787829662.589572:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829662.589576:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829662.589579:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829662.589580:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829662.589583:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787829662.589584:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787829662.589585:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589590:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829662.589593:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829662.589596:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.589597:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787829662.589599:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829662.589620:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716da40.
00000400:00000200:1.0:1787829662.589622:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000020:00000040:0.0:1787829662.589623:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:0.0:1787829662.589629:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1787829662.589630:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
02000000:00000001:0.0:1787829662.589631:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589634:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829662.589637:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829662.589639:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589641:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:0.0:1787829662.589643:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531000.
00000100:00000001:0.0:1787829662.589648:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829662.589650:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829662.589655:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829662.589681:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829662.589685:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.589694:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589703:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829662.589704:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589705:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787829662.589713:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000226d20b5] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589714:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.589741:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829662.589745:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589746:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298483373
00000100:00000040:0.0:1787829662.589750:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3820)
00000100:00000010:1.0:1787829662.589753:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63de0.
00000100:00000001:1.0:1787829662.589762:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829662.589768:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298483373
00000100:00000010:1.0:1787829662.589770:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63420.
00000100:00000001:1.0:1787829662.589772:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829662.589810:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829662.589814:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589816:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829662.589820:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662]  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829662.589846:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829662.589848:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829662.589850:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829662.589855:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829662.589857:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829662.589860:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829662.589864:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829662.589870:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589876:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829662.589878:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.589879:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.589881:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829662.589884:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589889:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829662.589894:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589900:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829662.589902:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.589903:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.589904:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.589906:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829662.589908:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829662.589910:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829662.589913:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829662.589914:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829662.589918:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829662.589924:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327366528:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829662.589928:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.589930:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.589932:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327366528/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589943:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829662.589946:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829662.589949:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829662.589951:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829662.589953:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829662.589954:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829662.589956:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829662.589958:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829662.589962:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829662.589964:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829662.589965:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829662.589968:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829662.589969:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.589971:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.589979:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829662.589981:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.589982:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829662.589984:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829662.589985:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829662.589988:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.589993:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829662.589994:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829662.589996:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829662.589998:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829662.589999:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829662.590017:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366656:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829662.590019:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829662.590020:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829662.590021:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829662.590022:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829662.590025:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829662.590026:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.590029:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829662.590032:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829662.590035:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829662.590037:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366656, portal 10
00000100:00000001:1.0:1787829662.590039:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829662.590040:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829662.590045:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590050:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829662.590051:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829662.590053:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829662.590055:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327366656, offset 0
00000400:00000200:1.0:1787829662.590057:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829662.590060:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829662.590063:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829662.590080:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829662.590083:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829662.590085:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829662.590087:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829662.590090:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40636:40636:256:4294967295] 192.168.203.116@tcp LPNI seq info [40636:40636:8:4294967295]
00000400:00000200:1.0:1787829662.590092:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829662.590098:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829662.590102:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829662.590104:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829662.590107:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829662.590110:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829662.590114:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829662.590148:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829662.590151:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829662.590152:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829662.590154:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829662.590157:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829662.590160:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829662.590161:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829662.590164:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829662.590166:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590172:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.590174:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590179:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590180:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829662.590182:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829662.590185:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829662.590187:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829662.590188:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298482699
00000100:00000001:1.0:1787829662.590190:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298483373
00000100:00000001:1.0:1787829662.590192:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298482699
00000100:00000001:1.0:1787829662.590193:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298483373
00000100:00000001:1.0:1787829662.590195:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829662.590197:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590198:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829662.590200:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662]  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829662.590205:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829662.590206:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829662.590207:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829662.590208:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829662.590209:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829662.590210:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829662.590212:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829662.590218:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590227:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829662.590228:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590229:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.590230:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829662.590233:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590237:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829662.590239:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590243:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829662.590245:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590245:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.590246:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590247:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829662.590249:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829662.590250:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829662.590252:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829662.590253:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829662.590256:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829662.590261:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366656:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829662.590264:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829662.590266:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590270:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829662.590271:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829662.590273:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829662.590274:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829662.590276:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829662.590278:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829662.590292:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829662.590293:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829662.590294:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829662.590295:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829662.590298:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829662.590299:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.590301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829662.590303:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829662.590306:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829662.590307:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366784, portal 4
00000100:00000001:1.0:1787829662.590309:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829662.590310:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829662.590314:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590318:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829662.590320:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829662.590322:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829662.590323:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327366784, offset 0
00000400:00000200:1.0:1787829662.590326:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829662.590328:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829662.590330:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829662.590331:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829662.590333:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829662.590335:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829662.590336:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829662.590339:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40637:40637:256:4294967295] 192.168.203.116@tcp LPNI seq info [40637:40637:8:4294967295]
00000400:00000200:1.0:1787829662.590342:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829662.590356:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829662.590359:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829662.590361:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829662.590364:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829662.590365:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829662.590368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829662.590370:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829662.590373:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829662.590374:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829662.590376:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829662.590378:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829662.590380:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829662.590381:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829662.590383:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829662.590385:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590390:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.590392:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590397:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590398:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829662.590399:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829662.590401:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829662.590403:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829662.590404:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298482706
00000100:00000001:1.0:1787829662.590406:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298483373
00000100:00000001:1.0:1787829662.590407:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298482706
00000100:00000001:1.0:1787829662.590409:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298483373
00000100:00000001:1.0:1787829662.590410:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829662.590411:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590412:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829662.590415:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662]  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829662.590420:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829662.590421:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829662.590423:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829662.590424:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829662.590425:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829662.590426:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829662.590428:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829662.590433:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590436:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829662.590437:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590438:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.590439:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829662.590441:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590445:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829662.590447:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590451:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829662.590453:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590454:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.590455:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590456:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829662.590457:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829662.590459:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829662.590461:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829662.590462:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829662.590465:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829662.590474:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829662.590478:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829662.590482:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590489:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829662.590491:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829662.590494:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829662.590495:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829662.590497:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829662.590500:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366912:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829662.590504:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829662.590506:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829662.590508:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829662.590510:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829662.590515:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829662.590517:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829662.590519:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829662.590521:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829662.590524:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829662.590526:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327366912, portal 4
00000100:00000001:1.0:1787829662.590527:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829662.590528:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829662.590533:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590538:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829662.590539:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829662.590541:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829662.590543:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327366912, offset 0
00000400:00000200:1.0:1787829662.590546:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829662.590548:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829662.590550:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829662.590553:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829662.590554:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829662.590556:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829662.590558:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829662.590560:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40638:40638:256:4294967295] 192.168.203.116@tcp LPNI seq info [40638:40638:8:4294967295]
00000400:00000200:1.0:1787829662.590563:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829662.590567:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829662.590570:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829662.590572:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829662.590575:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829662.590577:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829662.590579:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829662.590581:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829662.590584:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829662.590585:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829662.590587:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829662.590590:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829662.590592:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829662.590593:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829662.590595:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829662.590597:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590602:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.590604:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590609:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590610:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829662.590612:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829662.590614:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829662.590616:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829662.590618:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298482711
00000100:00000001:1.0:1787829662.590619:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298483373
00000100:00000001:1.0:1787829662.590621:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298482711
00000100:00000001:1.0:1787829662.590623:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298483373
00000100:00000001:1.0:1787829662.590624:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829662.590625:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590626:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829662.590628:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662]  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829662.590633:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829662.590634:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829662.590635:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829662.590636:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829662.590638:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829662.590639:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829662.590641:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829662.590646:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590650:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829662.590651:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590652:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.590653:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829662.590655:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590659:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829662.590661:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590666:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829662.590668:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590670:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829662.590671:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590672:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829662.590673:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829662.590675:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829662.590677:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829662.590678:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829662.590680:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829662.590685:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327366912:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829662.590687:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590688:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.590690:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327366656/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590694:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829662.590695:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829662.590696:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829662.590697:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829662.590698:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829662.590699:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829662.590701:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829662.590702:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829662.590704:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829662.590704:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829662.590706:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829662.590708:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829662.590708:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590710:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.590712:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327366784/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590737:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829662.590738:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829662.590740:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829662.590741:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829662.590742:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829662.590742:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829662.590743:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829662.590744:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:1.0:1787829662.590746:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829662.590746:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829662.590747:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829662.590749:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829662.590750:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590751:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829662.590754:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327366912/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829662.590758:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829662.590759:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829662.590760:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829662.590762:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829662.590762:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829662.590763:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829662.590764:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829662.590765:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829662.590766:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829662.590767:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829662.590768:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829662.590769:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829662.590770:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590771:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829662.590774:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829662.590774:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829662.590776:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829662.590777:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829663.100802:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829663.100810:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829663.100812:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829663.100818:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829663.100822:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829663.100841:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829663.100843:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829663.100845:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829663.100847:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829663.292748:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298483373, at 4298483392
00000100:00000040:3.0:1787829663.292748:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298483373, at 4298483392
00000100:00000010:3.0:1787829663.292764:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63420.
00000100:00000010:2.0:1787829663.292766:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63de0.
00010000:00000001:0.0:1787829663.420754:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829663.420759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829663.420764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829663.420773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829663.420776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829663.420777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829663.420840:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829663.420843:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829663.420846:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829664.124765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829664.124771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829664.124772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829664.124788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829664.124791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829664.124795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829664.124797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829664.124798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829664.124801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829664.444791:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829664.444792:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829664.444795:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444800:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829664.444801:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444803:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444805:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444809:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829664.444811:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829664.444811:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829664.444813:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444814:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829664.444815:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444816:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444817:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829664.444819:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829664.444820:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829664.444851:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829664.444854:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829664.444857:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829665.148740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829665.148746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829665.148747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829665.148751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829665.148754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829665.148757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829665.148759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829665.148761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829665.148763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829665.468805:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829665.468819:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829665.468825:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829665.468826:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829665.468830:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829665.468832:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829665.468868:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829665.468871:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829665.468874:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829666.172820:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829666.172827:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829666.172830:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829666.172835:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829666.172839:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829666.172843:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829666.172846:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829666.172849:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829666.172852:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829666.492729:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829666.492735:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829666.492738:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829666.492740:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829666.492742:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829666.492743:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829666.492800:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829666.492804:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829666.492807:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829667.196723:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829667.196728:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829667.196730:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829667.196734:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829667.196736:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829667.196739:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829667.196741:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829667.196743:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829667.196745:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829667.516769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829667.516776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829667.516783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829667.516784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829667.516788:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829667.516789:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829667.516862:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829667.516866:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829667.516870:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829667.708790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829667.708799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829667.708801:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829667.708804:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829667.708807:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829667.708813:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3815
00000400:00000200:2.0:1787829667.708820:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829667.708825:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829667.708828:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829667.708830:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708832:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829667.708834:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708836:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829667.708839:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829667.708840:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829667.708842:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829667.708844:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829667.708846:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829667.708848:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829667.708850:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708851:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829667.708853:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708854:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829667.708856:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829667.708858:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708860:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829667.708862:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829667.708865:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829667.708866:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.708868:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829667.708870:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829667.708878:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829667.708880:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829667.708884:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829667.708885:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.708887:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829667.708889:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829667.708892:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829667.708898:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:2.0:1787829667.708903:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829667.708911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829667.708918:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829667.708921:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829667.708925:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829667.708944:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829667.708947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829667.708950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829667.708951:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829667.708953:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829667.708957:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829667.708959:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3815
00000400:00000200:2.0:1787829667.708963:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829667.708966:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829667.708969:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829667.708973:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708976:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829667.708979:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.708982:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829667.708983:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787829667.708985:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787829667.708988:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1787829667.708990:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:1.0:1787829667.708992:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787829667.708994:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829667.708998:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000040:1.0:1787829667.709000:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829667.709004:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829667.709008:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829667.709012:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709013:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829667.709022:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787829667.709023:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829667.709025:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709026:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:1.0:1787829667.709030:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787829667.709030:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:1.0:1787829667.709034:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1787829667.709035:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829667.709038:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709039:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829667.709043:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787829667.709044:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327367040:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829667.709049:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829667.709050:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829667.709052:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829667.709054:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.709055:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709057:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829667.709059:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:2.0:1787829667.709062:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000010:1.0:1787829667.709065:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
00000008:00000001:2.0:1787829667.709066:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.709068:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709070:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.709075:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829667.709079:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000100:00000010:2.0:1787829667.709079:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787829667.709083:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:1.0:1787829667.709087:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
02000000:00000001:2.0:1787829667.709090:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829667.709092:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829667.709094:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367040, portal 25
00000100:00000001:2.0:1787829667.709094:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829667.709095:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829667.709098:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709099:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829667.709103:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
02000000:00000010:2.0:1787829667.709106:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000040:1.0:1787829667.709108:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829667.709112:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709119:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:2.0:1787829667.709120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787829667.709125:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829667.709130:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000001:2.0:1787829667.709131:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787829667.709134:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327367040, offset 0
00000100:00000001:2.0:1787829667.709135:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829667.709141:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829667.709144:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829667.709154:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829667.709156:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.709161:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:2.0:1787829667.709162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829667.709166:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787829667.709170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1787829667.709174:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787829667.709175:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787829667.709180:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1787829667.709180:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829667.709187:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709187:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829667.709194:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3815
00000400:00000200:1.0:1787829667.709196:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40639:40639:256:4294967295] 192.168.203.116@tcp LPNI seq info [40639:40639:8:4294967295]
00000400:00000200:2.0:1787829667.709201:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.709208:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829667.709208:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829667.709214:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829667.709216:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709218:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829667.709221:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709225:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829667.709229:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787829667.709232:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787829667.709232:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829667.709236:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829667.709240:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829667.709242:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:1.0:1787829667.709247:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829667.709247:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:1.0:1787829667.709254:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00000040:2.0:1787829667.709262:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709264:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829667.709266:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709268:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829667.709269:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829667.709272:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709276:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829667.709278:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:1.0:1787829667.709279:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829667.709282:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:1.0:1787829667.709286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1787829667.709286:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709291:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829667.709295:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.1:1787829667.709297:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00000001:2.0:1787829667.709300:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829667.709302:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1787829667.709305:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787829667.709310:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000010:2.0:1787829667.709311:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000800:00000010:1.0:1787829667.709314:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000020:00000040:2.0:1787829667.709317:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:1.0:1787829667.709321:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787829667.709322:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829667.709326:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709331:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787829667.709332:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1787829667.709333:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829667.709337:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.709340:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
02000000:00000010:2.0:1787829667.709342:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000400:00000010:1.0:1787829667.709345:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:2.0:1787829667.709347:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709351:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:2.0:1787829667.709354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787829667.709360:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829667.709364:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829667.709368:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829667.709375:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709381:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.709387:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709411:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709411:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709416:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:2.0:1787829667.709418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829667.709422:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829667.709424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787829667.709428:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829667.709428:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829667.709432:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829667.709433:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298488492
00000100:00000001:2.0:1787829667.709437:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829667.709441:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3815
00000100:00000010:1.0:1787829667.709442:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63ea0.
00000400:00000200:2.0:1787829667.709447:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829667.709450:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829667.709451:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:2.0:1787829667.709452:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829667.709458:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709459:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829667.709460:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298488492
00000100:00000040:2.0:1787829667.709462:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:1.0:1787829667.709467:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e635a0.
00000100:00000001:2.0:1787829667.709472:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829667.709473:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000001:2.0:1787829667.709476:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829667.709479:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:1.0:1787829667.709480:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1787829667.709480:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829667.709484:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829667.709486:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829667.709487:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:1.0:1787829667.709492:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:2.0:1787829667.709492:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829667.709497:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709498:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1787829667.709502:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667]  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829667.709502:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709506:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829667.709510:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829667.709514:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829667.709520:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829667.709523:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829667.709527:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829667.709529:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787829667.709530:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1787829667.709531:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000001:1.0:1787829667.709535:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000008:00100000:2.0:1787829667.709535:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787829667.709539:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000008:00000001:2.0:1787829667.709540:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.709546:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829667.709548:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:2.0:1787829667.709550:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709551:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000010:2.0:1787829667.709555:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000100:00000200:1.0:1787829667.709557:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829667.709559:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829667.709564:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829667.709566:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829667.709571:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829667.709573:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:1.0:1787829667.709576:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829667.709576:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829667.709582:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:1.0:1787829667.709588:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829667.709589:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709591:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709594:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.709597:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829667.709597:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1787829667.709602:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829667.709606:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829667.709610:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709616:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829667.709618:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829667.709623:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829667.709633:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709638:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1787829667.709639:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3825)
00000100:00000001:1.0:1787829667.709640:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709644:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.709645:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709647:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829667.709649:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829667.709651:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829667.709681:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829667.709684:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829667.709687:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829667.709693:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327367040:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829667.709696:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709698:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.709700:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327367040/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709704:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829667.709706:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829667.709708:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829667.709711:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829667.709712:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829667.709713:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829667.709715:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829667.709717:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:1.0:1787829667.709720:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829667.709721:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829667.709722:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:1.0:1787829667.709725:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829667.709726:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709728:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709733:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829667.709734:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709736:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829667.709737:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829667.709738:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829667.709740:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709749:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829667.709765:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.709767:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829667.709769:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829667.709770:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829667.709773:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367168:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829667.709775:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829667.709777:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829667.709778:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.709779:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829667.709781:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829667.709783:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.709786:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829667.709788:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829667.709790:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829667.709793:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367168, portal 10
00000100:00000001:1.0:1787829667.709794:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829667.709795:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:1.0:1787829667.709800:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709804:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829667.709806:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829667.709807:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829667.709809:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327367168, offset 0
00000400:00000200:1.0:1787829667.709812:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829667.709815:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.709818:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829667.709820:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829667.709822:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829667.709824:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829667.709825:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829667.709828:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40640:40640:256:4294967295] 192.168.203.116@tcp LPNI seq info [40640:40640:8:4294967295]
00000400:00000200:1.0:1787829667.709831:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.709836:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829667.709839:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829667.709841:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829667.709844:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829667.709846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829667.709849:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829667.709851:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829667.709853:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829667.709855:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829667.709857:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829667.709859:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829667.709861:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829667.709862:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829667.709864:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829667.709866:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709872:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.709874:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709878:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709879:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829667.709881:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829667.709884:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829667.709886:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829667.709888:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298487819
00000100:00000001:1.0:1787829667.709890:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298488492
00000100:00000001:1.0:1787829667.709891:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298487819
00000100:00000001:1.0:1787829667.709893:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298488492
00000100:00000001:1.0:1787829667.709894:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829667.709896:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.709897:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829667.709899:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667]  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829667.709904:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829667.709905:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829667.709906:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829667.709908:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829667.709909:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829667.709910:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829667.709912:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829667.709918:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709922:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829667.709922:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709923:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.709924:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829667.709926:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709930:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829667.709933:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709937:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829667.709938:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709939:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.709940:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.709941:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829667.709943:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829667.709944:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829667.709946:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829667.709948:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829667.709951:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829667.709955:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367168:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829667.709958:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829667.709960:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709963:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829667.709964:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.709965:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829667.709967:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829667.709968:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829667.709969:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367296:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829667.709971:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829667.709972:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829667.709973:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.709974:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829667.709976:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829667.709977:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.709980:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829667.709981:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829667.709983:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829667.709985:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367296, portal 4
00000100:00000001:1.0:1787829667.709986:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829667.709987:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:1.0:1787829667.709991:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.709995:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829667.709997:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829667.709998:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829667.710000:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327367296, offset 0
00000400:00000200:1.0:1787829667.710002:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829667.710005:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.710007:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829667.710009:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829667.710011:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829667.710012:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829667.710014:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829667.710016:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40641:40641:256:4294967295] 192.168.203.116@tcp LPNI seq info [40641:40641:8:4294967295]
00000400:00000200:1.0:1787829667.710018:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.710022:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829667.710025:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829667.710027:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829667.710029:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829667.710031:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829667.710033:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829667.710035:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829667.710037:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829667.710039:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829667.710040:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829667.710043:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829667.710044:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829667.710045:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829667.710047:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829667.710049:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710054:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.710057:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710061:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.710062:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829667.710064:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829667.710066:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829667.710068:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829667.710070:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298487826
00000100:00000001:1.0:1787829667.710071:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298488492
00000100:00000001:1.0:1787829667.710073:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298487826
00000100:00000001:1.0:1787829667.710074:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298488492
00000100:00000001:1.0:1787829667.710075:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829667.710076:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.710078:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829667.710080:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667]  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829667.710084:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829667.710085:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829667.710086:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829667.710087:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829667.710088:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829667.710090:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829667.710091:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829667.710097:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710101:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829667.710101:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710102:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.710103:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829667.710105:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710109:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829667.710111:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710116:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829667.710118:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710119:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.710120:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710121:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829667.710122:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829667.710123:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829667.710125:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829667.710126:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829667.710128:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829667.710133:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367296:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829667.710135:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829667.710137:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710141:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829667.710142:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.710143:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829667.710144:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829667.710145:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829667.710147:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367424:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829667.710148:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829667.710149:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829667.710150:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829667.710152:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829667.710154:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829667.710155:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829667.710157:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829667.710159:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829667.710161:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829667.710162:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367424, portal 4
00000100:00000001:1.0:1787829667.710164:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829667.710165:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:1.0:1787829667.710168:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710172:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829667.710173:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829667.710175:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829667.710176:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327367424, offset 0
00000400:00000200:1.0:1787829667.710179:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829667.710180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.710182:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829667.710184:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829667.710186:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829667.710187:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829667.710189:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829667.710191:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40642:40642:256:4294967295] 192.168.203.116@tcp LPNI seq info [40642:40642:8:4294967295]
00000400:00000200:1.0:1787829667.710193:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829667.710197:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829667.710200:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829667.710202:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829667.710204:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829667.710205:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829667.710208:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829667.710210:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829667.710212:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829667.710214:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829667.710215:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829667.710217:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829667.710219:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829667.710220:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829667.710222:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829667.710224:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710229:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.710231:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710235:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.710236:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829667.710238:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829667.710240:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829667.710242:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829667.710243:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298487831
00000100:00000001:1.0:1787829667.710245:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298488492
00000100:00000001:1.0:1787829667.710246:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298487831
00000100:00000001:1.0:1787829667.710248:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298488492
00000100:00000001:1.0:1787829667.710249:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829667.710250:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.710251:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829667.710253:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667]  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829667.710258:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829667.710258:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829667.710259:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829667.710260:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829667.710261:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829667.710263:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829667.710264:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829667.710269:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710278:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829667.710280:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710282:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.710283:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829667.710287:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710294:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829667.710298:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710305:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829667.710308:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710309:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829667.710311:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710313:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829667.710315:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829667.710318:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829667.710321:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829667.710324:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829667.710328:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829667.710335:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367424:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829667.710338:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710339:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.710341:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327367168/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710346:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829667.710347:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829667.710349:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:1.0:1787829667.710350:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829667.710352:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829667.710353:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829667.710354:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829667.710355:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:1.0:1787829667.710357:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829667.710358:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829667.710360:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:1.0:1787829667.710362:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829667.710363:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710365:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.710368:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327367296/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710375:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829667.710377:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829667.710378:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829667.710381:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829667.710382:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829667.710384:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829667.710386:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829667.710387:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:1.0:1787829667.710389:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829667.710390:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829667.710392:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:1.0:1787829667.710394:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829667.710396:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710397:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829667.710401:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327367424/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829667.710408:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829667.710410:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829667.710412:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829667.710414:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829667.710415:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829667.710416:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829667.710418:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829667.710419:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:1.0:1787829667.710420:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829667.710421:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829667.710422:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:1.0:1787829667.710423:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829667.710424:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710425:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829667.710428:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829667.710429:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829667.710431:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829667.710433:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829668.220813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829668.220818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829668.220820:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829668.220823:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829668.220827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829668.220830:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829668.220832:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829668.220834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829668.220837:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829668.412775:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298488492, at 4298488512
00000100:00000040:2.0:1787829668.412776:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298488492, at 4298488512
00000100:00000010:3.0:1787829668.412791:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e635a0.
00000100:00000010:2.0:1787829668.412793:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ea0.
00010000:00000001:0.0:1787829668.540763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829668.540769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829668.540775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829668.540776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829668.540779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829668.540781:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829668.540799:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829668.540802:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829668.540805:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829669.244735:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829669.244757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829669.244759:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829669.244764:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829669.244767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829669.244771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829669.244773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829669.244792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829669.244794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829669.564763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829669.564798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829669.564804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829669.564806:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829669.564810:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829669.564813:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829669.564816:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829669.564819:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829669.564823:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829669.564825:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829669.564827:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829669.564830:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829669.564888:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829669.564892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829669.564895:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829670.268808:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829670.268815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829670.268818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829670.268822:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829670.268826:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829670.268830:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829670.268833:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829670.268836:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829670.268839:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829670.588821:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829670.588827:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829670.588832:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829670.588833:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829670.588836:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829670.588837:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829670.588893:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829670.588897:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829670.588901:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829671.292711:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829671.292717:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829671.292719:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829671.292722:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829671.292737:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829671.292740:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829671.292743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829671.292745:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829671.292747:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829671.612806:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829671.612814:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829671.612819:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829671.612821:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829671.612824:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829671.612825:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829671.612863:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829671.612867:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829671.612872:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829672.316784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829672.316792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829672.316795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829672.316800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829672.316804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829672.316807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829672.316810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829672.316813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829672.316817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829672.636815:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829672.636825:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829672.636834:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829672.636835:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829672.636839:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829672.636840:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829672.636889:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829672.636895:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829672.636899:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829672.828759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829672.828767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829672.828770:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829672.828773:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829672.828776:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829672.828782:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3820
00000400:00000200:2.0:1787829672.828789:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829672.828794:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829672.828797:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829672.828799:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828802:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829672.828804:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828806:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829672.828809:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829672.828810:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829672.828812:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829672.828815:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829672.828817:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829672.828818:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829672.828820:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828822:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829672.828823:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828825:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829672.828827:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829672.828829:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828831:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829672.828833:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829672.828835:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829672.828837:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.828839:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829672.828840:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829672.828848:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787829672.828850:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829672.828853:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829672.828855:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.828856:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829672.828858:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829672.828862:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829672.828868:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829672.828873:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829672.828880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829672.828889:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829672.828892:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829672.828895:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829672.828916:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829672.828918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829672.828920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829672.828921:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829672.828923:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829672.828924:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829672.828927:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3820
00000400:00000200:2.0:1787829672.828929:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829672.828931:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829672.828932:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829672.828935:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828936:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829672.828938:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828940:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829672.828942:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829672.828942:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829672.828943:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829672.828945:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829672.828947:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829672.828948:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829672.828950:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828952:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829672.828954:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828955:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829672.828957:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829672.828961:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829672.828963:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829672.828965:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829672.828970:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829672.828972:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.828974:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829672.828977:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829672.828979:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.828981:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.828985:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000010:2.0:1787829672.828985:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829672.828995:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:0.0:1787829672.828996:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787829672.828999:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829672.829000:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.829001:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787829672.829003:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829672.829005:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829672.829006:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829672.829010:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000040:0.0:1787829672.829015:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829672.829016:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829672.829028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829672.829042:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829672.829049:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829061:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829672.829063:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787829672.829064:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.829068:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787829672.829073:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829672.829077:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829672.829090:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327367552:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829672.829102:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787829672.829103:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.829111:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1787829672.829111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829672.829115:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.829120:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1787829672.829126:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829672.829129:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829672.829132:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000010:0.0:1787829672.829133:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829672.829138:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829141:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829672.829150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829672.829151:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3820
00000400:00000010:0.0:1787829672.829157:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000200:2.0:1787829672.829164:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829672.829168:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000400:00000200:2.0:1787829672.829173:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829672.829179:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1787829672.829180:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367552, portal 25
00000100:00000001:0.0:1787829672.829186:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1787829672.829186:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829672.829190:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402430400 : -116696307121216 : ffff95dd861143c0)
00000100:00000001:2.0:1787829672.829193:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829672.829201:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829672.829203:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829672.829217:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787829672.829217:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829672.829226:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829672.829228:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:0.0:1787829672.829229:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829672.829237:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327367552, offset 0
00000400:00000200:0.0:1787829672.829242:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787829672.829243:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829672.829248:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829672.829251:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829672.829255:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787829672.829256:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829672.829263:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829672.829266:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:2.0:1787829672.829271:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829672.829273:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787829672.829281:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829672.829283:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000040:2.0:1787829672.829288:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829672.829293:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829672.829293:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829672.829302:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829672.829306:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:2.0:1787829672.829307:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829672.829314:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829672.829316:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829672.829323:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829672.829324:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40643:40643:256:4294967295] 192.168.203.116@tcp LPNI seq info [40643:40643:8:4294967295]
02000000:00000001:2.0:1787829672.829330:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829336:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829672.829340:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829672.829349:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00000001:2.0:1787829672.829350:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829357:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829672.829360:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787829672.829363:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829672.829369:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829672.829374:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829672.829375:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787829672.829381:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829672.829383:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829672.829386:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829672.829387:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000010:2.0:1787829672.829402:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000800:00000010:0.0:1787829672.829403:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00000001:2.0:1787829672.829416:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829672.829422:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:2.0:1787829672.829424:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829672.829430:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787829672.829444:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829672.829453:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829672.829461:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829672.829468:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787829672.829469:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:0.0:1787829672.829486:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.829492:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000100:00000001:2.0:1787829672.829494:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829672.829499:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829672.829503:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1787829672.829503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829672.829508:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829672.829513:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829672.829515:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829672.829517:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829672.829522:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3820
00000400:00000200:0.0:1787829672.829523:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000200:2.0:1787829672.829534:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829672.829535:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000400:00000200:2.0:1787829672.829545:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829672.829547:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1787829672.829548:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1787829672.829555:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829672.829556:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.829567:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829672.829574:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.829581:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829672.829586:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829672.829589:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829672.829593:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829672.829594:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829672.829597:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829672.829599:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:2.0:1787829672.829599:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829672.829606:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.829611:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829672.829613:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829672.829615:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829672.829621:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829672.829629:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829672.829641:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829672.829654:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829672.829676:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829672.829679:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829672.829680:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829682:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829672.829684:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829672.829687:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829688:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829672.829691:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787829672.829693:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829672.829695:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829672.829695:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.829697:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829672.829697:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829672.829701:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829672.829708:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787829672.829708:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829672.829713:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829672.829729:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.829733:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829672.829734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829672.829744:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829672.829747:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829672.829749:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829672.829756:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.829757:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298493613
00000100:00000010:0.0:1787829672.829771:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc180.
00000100:00000001:2.0:1787829672.829771:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829672.829783:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3830)
00000100:00000001:0.0:1787829672.829785:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829672.829795:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298493613
00000100:00000010:0.0:1787829672.829800:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc5a0.
00000100:00000001:0.0:1787829672.829804:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829672.829808:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829672.829810:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.829813:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829672.829818:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672]  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829672.829826:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829672.829828:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829672.829829:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829672.829832:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829672.829833:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829672.829836:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829672.829838:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829672.829845:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.829849:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829672.829850:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.829852:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.829854:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829672.829857:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.829862:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829672.829866:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.829873:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829672.829876:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.829879:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.829880:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.829883:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829672.829885:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829672.829889:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829672.829894:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829672.829898:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829672.829903:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829672.829913:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327367552:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829672.829919:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.829922:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.829924:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861143c0 x1874671327367552/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.829929:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829672.829932:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829672.829935:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829672.829937:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829672.829939:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829672.829941:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829672.829943:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829672.829945:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829672.829949:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829672.829951:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829672.829952:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861143c0.
00000100:00000001:0.0:1787829672.829955:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829672.829956:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.829957:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.829965:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829672.829966:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.829968:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829672.829969:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829672.829970:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829672.829973:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.829978:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829672.829980:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.829981:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829672.829983:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829672.829984:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829672.829986:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367680:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829672.829989:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829672.829990:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829672.829992:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.829993:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829672.829996:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829672.829997:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829672.830000:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829672.830002:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829672.830005:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829672.830008:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367680, portal 10
00000100:00000001:0.0:1787829672.830009:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829672.830010:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:0.0:1787829672.830015:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830020:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829672.830022:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829672.830023:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829672.830025:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327367680, offset 0
00000400:00000200:0.0:1787829672.830028:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829672.830030:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829672.830034:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829672.830036:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829672.830038:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829672.830040:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829672.830042:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829672.830045:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40644:40644:256:4294967295] 192.168.203.116@tcp LPNI seq info [40644:40644:8:4294967295]
00000400:00000200:0.0:1787829672.830048:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829672.830052:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829672.830056:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.830058:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829672.830061:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829672.830063:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829672.830067:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829672.830069:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829672.830072:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.830073:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829672.830076:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829672.830078:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829672.830080:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829672.830081:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829672.830083:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829672.830085:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830091:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.830092:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830097:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830098:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829672.830100:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829672.830104:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829672.830106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829672.830108:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298492939
00000100:00000001:0.0:1787829672.830110:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298493613
00000100:00000001:0.0:1787829672.830112:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298492939
00000100:00000001:0.0:1787829672.830113:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298493613
00000100:00000001:0.0:1787829672.830115:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829672.830117:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830118:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829672.830121:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672]  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829672.830126:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829672.830126:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829672.830127:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829672.830129:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829672.830130:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829672.830131:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829672.830133:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829672.830139:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830143:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829672.830144:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830145:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.830146:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829672.830148:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830152:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829672.830154:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830158:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829672.830160:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830161:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.830162:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830163:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829672.830164:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829672.830167:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829672.830169:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829672.830171:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829672.830173:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829672.830178:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367680:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829672.830180:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829672.830183:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830187:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829672.830188:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.830189:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829672.830190:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829672.830192:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829672.830193:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367808:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829672.830195:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829672.830196:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829672.830197:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.830198:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829672.830200:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829672.830202:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829672.830204:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829672.830205:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829672.830208:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829672.830210:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367808, portal 4
00000100:00000001:0.0:1787829672.830211:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829672.830212:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:0.0:1787829672.830216:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830225:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829672.830227:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829672.830228:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829672.830230:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327367808, offset 0
00000400:00000200:0.0:1787829672.830232:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829672.830234:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829672.830236:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829672.830238:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829672.830239:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829672.830241:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829672.830242:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829672.830245:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40645:40645:256:4294967295] 192.168.203.116@tcp LPNI seq info [40645:40645:8:4294967295]
00000400:00000200:0.0:1787829672.830247:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829672.830252:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829672.830255:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.830256:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829672.830259:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829672.830261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829672.830264:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829672.830266:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829672.830268:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.830270:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829672.830272:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829672.830274:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829672.830275:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829672.830277:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829672.830278:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829672.830280:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830285:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.830288:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830300:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830302:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829672.830305:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829672.830308:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829672.830312:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829672.830314:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298492946
00000100:00000001:0.0:1787829672.830317:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298493613
00000100:00000001:0.0:1787829672.830320:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298492946
00000100:00000001:0.0:1787829672.830322:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298493613
00000100:00000001:0.0:1787829672.830324:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829672.830327:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830329:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829672.830333:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672]  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829672.830342:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829672.830344:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829672.830345:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829672.830347:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829672.830348:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829672.830350:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829672.830352:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829672.830358:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830362:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829672.830363:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830364:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.830365:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829672.830367:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830372:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829672.830375:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830379:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829672.830381:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830382:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.830382:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830383:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829672.830384:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829672.830386:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829672.830388:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829672.830389:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829672.830391:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829672.830396:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367808:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829672.830398:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829672.830401:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830406:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829672.830407:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.830409:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829672.830410:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829672.830412:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829672.830413:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367936:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829672.830416:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829672.830417:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829672.830419:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829672.830420:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829672.830423:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829672.830424:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829672.830427:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829672.830429:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829672.830431:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829672.830433:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327367936, portal 4
00000100:00000001:0.0:1787829672.830434:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829672.830435:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:0.0:1787829672.830439:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830444:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829672.830446:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829672.830447:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829672.830449:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327367936, offset 0
00000400:00000200:0.0:1787829672.830451:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829672.830453:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829672.830456:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829672.830458:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829672.830460:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829672.830461:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829672.830463:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829672.830465:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40646:40646:256:4294967295] 192.168.203.116@tcp LPNI seq info [40646:40646:8:4294967295]
00000400:00000200:0.0:1787829672.830468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829672.830473:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829672.830476:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.830477:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829672.830480:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829672.830482:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829672.830484:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829672.830486:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829672.830489:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829672.830491:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829672.830493:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829672.830496:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829672.830498:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829672.830499:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829672.830500:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829672.830502:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830508:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.830510:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830514:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830515:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829672.830516:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829672.830518:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829672.830520:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829672.830521:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298492951
00000100:00000001:0.0:1787829672.830523:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298493613
00000100:00000001:0.0:1787829672.830524:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298492951
00000100:00000001:0.0:1787829672.830526:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298493613
00000100:00000001:0.0:1787829672.830528:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829672.830529:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830530:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829672.830532:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672]  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829672.830537:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829672.830538:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829672.830539:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829672.830540:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829672.830541:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829672.830542:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829672.830544:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829672.830550:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830554:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829672.830554:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830556:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.830556:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829672.830558:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830562:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829672.830564:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830568:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829672.830570:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830571:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829672.830572:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830574:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829672.830576:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829672.830578:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829672.830581:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829672.830583:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829672.830587:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829672.830595:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327367936:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829672.830599:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830602:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.830606:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327367680/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830614:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829672.830616:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829672.830618:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829672.830621:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829672.830622:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829672.830624:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829672.830626:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829672.830627:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829672.830629:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829672.830630:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829672.830632:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:0.0:1787829672.830634:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829672.830635:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830637:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.830639:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327367808/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830644:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829672.830645:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829672.830646:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829672.830647:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829672.830648:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829672.830649:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829672.830650:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829672.830651:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829672.830652:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829672.830653:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829672.830653:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:0.0:1787829672.830674:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829672.830676:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830677:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829672.830679:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327367936/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829672.830683:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829672.830684:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829672.830686:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829672.830687:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829672.830688:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829672.830689:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829672.830691:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829672.830691:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829672.830693:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829672.830693:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829672.830694:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:0.0:1787829672.830696:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829672.830696:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830697:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829672.830701:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829672.830702:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829672.830704:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829672.830706:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829673.340728:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829673.340735:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829673.340737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829673.340741:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829673.340743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829673.340746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829673.340748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829673.340749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829673.340752:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829673.532740:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298493613, at 4298493632
00000100:00000040:3.0:1787829673.532741:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298493613, at 4298493632
00000100:00000010:3.0:1787829673.532755:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000010:2.0:1787829673.532758:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00010000:00000001:0.0:1787829673.660750:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829673.660756:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829673.660760:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829673.660761:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829673.660764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829673.660765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829673.660820:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829673.660823:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829673.660825:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829674.364782:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829674.364791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829674.364794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829674.364799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829674.364804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829674.364808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829674.364810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829674.364813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829674.364816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684785:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829674.684792:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829674.684794:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829674.684797:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684801:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829674.684803:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684805:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684806:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684809:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829674.684810:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829674.684811:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829674.684812:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684813:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829674.684814:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684815:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684817:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829674.684819:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829674.684820:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829674.684836:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829674.684838:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829674.684840:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829675.388722:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829675.388728:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829675.388729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829675.388734:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829675.388737:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829675.388744:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829675.388747:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829675.388749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829675.388751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829675.708769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829675.708775:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829675.708781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829675.708783:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829675.708786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829675.708788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829675.708853:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829675.708857:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829675.708861:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829676.412769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829676.412775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829676.412777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829676.412781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829676.412783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829676.412787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829676.412789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829676.412792:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829676.412794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829676.732763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829676.732768:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829676.732772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829676.732774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829676.732777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829676.732778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829676.732828:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829676.732831:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829676.732834:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829677.436727:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829677.436734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829677.436736:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829677.436740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829677.436743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829677.436745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829677.436747:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829677.436749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829677.436751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829677.756769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829677.756776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829677.756782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829677.756784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829677.756788:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829677.756789:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829677.756810:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829677.756813:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829677.756816:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829677.948752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829677.948761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829677.948763:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829677.948766:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829677.948769:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829677.948775:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3825
00000400:00000200:3.0:1787829677.948782:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829677.948787:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829677.948790:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829677.948792:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948794:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.948796:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948798:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829677.948801:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829677.948802:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829677.948805:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829677.948807:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829677.948809:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829677.948810:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829677.948812:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948814:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.948815:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948816:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829677.948818:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829677.948821:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948822:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829677.948825:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829677.948827:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829677.948829:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.948831:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829677.948832:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829677.948839:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829677.948843:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829677.948846:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829677.948847:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.948849:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829677.948852:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829677.948855:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829677.948861:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829677.948866:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829677.948873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829677.948881:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829677.948884:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829677.948888:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829677.948906:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829677.948908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829677.948911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829677.948912:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829677.948913:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829677.948915:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829677.948916:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3825
00000400:00000200:3.0:1787829677.948919:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829677.948920:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829677.948922:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829677.948924:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948925:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.948927:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948928:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829677.948930:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829677.948931:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829677.948932:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829677.948934:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829677.948935:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829677.948937:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829677.948940:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948941:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.948942:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948944:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829677.948946:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829677.948950:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829677.948952:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829677.948954:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829677.948955:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:3.0:1787829677.948960:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829677.948962:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:3.0:1787829677.948964:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.948967:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829677.948968:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829677.948975:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829677.948979:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829677.948980:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829677.948982:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829677.948989:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829677.948993:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:0.0:1787829677.948999:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787829677.949000:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829677.949003:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829677.949004:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949009:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829677.949010:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787829677.949011:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829677.949014:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949016:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829677.949019:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829677.949019:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829677.949025:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829677.949026:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327368064:192.168.203.116@tcp:250:kworker.0
00000100:00100000:3.0:1787829677.949032:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829677.949033:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829677.949041:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787829677.949041:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829677.949045:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949046:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829677.949049:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829677.949052:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829677.949060:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
00000100:00000001:3.0:1787829677.949064:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949066:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829677.949068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829677.949071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829677.949073:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829677.949074:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829677.949076:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829677.949077:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829677.949081:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3825
00000400:00000010:0.0:1787829677.949085:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000200:3.0:1787829677.949085:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829677.949091:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829677.949093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000010:0.0:1787829677.949094:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000040:3.0:1787829677.949096:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949099:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.949101:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949102:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829677.949103:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368064, portal 25
00000020:00000001:3.0:1787829677.949104:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829677.949107:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000001:0.0:1787829677.949108:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:3.0:1787829677.949110:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829677.949112:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000020:00000040:3.0:1787829677.949113:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829677.949117:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829677.949119:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:0.0:1787829677.949123:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829677.949123:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949127:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.949130:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949134:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829677.949138:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:0.0:1787829677.949142:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1787829677.949143:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949147:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829677.949150:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829677.949151:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1787829677.949154:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
02000000:00000001:3.0:1787829677.949155:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1787829677.949158:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327368064, offset 0
02000000:00000001:3.0:1787829677.949159:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949164:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829677.949166:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829677.949170:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000008:00000001:3.0:1787829677.949171:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949174:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829677.949179:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000400:00000200:0.0:1787829677.949180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:3.0:1787829677.949182:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829677.949185:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829677.949188:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:3.0:1787829677.949188:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949190:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829677.949194:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829677.949195:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829677.949196:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:3.0:1787829677.949199:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000400:00000200:0.0:1787829677.949203:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:3.0:1787829677.949204:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829677.949209:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00100000:3.0:1787829677.949211:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829677.949215:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829677.949224:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40647:40647:256:4294967295] 192.168.203.116@tcp LPNI seq info [40647:40647:8:4294967295]
00000100:00000001:3.0:1787829677.949224:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829677.949227:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829677.949233:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829677.949234:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829677.949242:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829677.949246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829677.949251:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829677.949256:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829677.949259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000010:0.0:1787829677.949261:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00000001:3.0:1787829677.949262:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829677.949264:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829677.949267:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.0:1787829677.949272:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829677.949272:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3825
00000400:00000200:3.0:1787829677.949277:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.0:1787829677.949279:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000400:00000200:3.0:1787829677.949284:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829677.949289:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1787829677.949290:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829677.949296:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000040:3.0:1787829677.949298:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949302:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829677.949304:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949306:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829677.949309:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829677.949311:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000100:0.0:1787829677.949316:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829677.949316:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829677.949320:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000010:0.0:1787829677.949321:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000020:00000001:3.0:1787829677.949326:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829677.949328:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:3.0:1787829677.949330:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829677.949335:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1787829677.949337:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949340:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829677.949341:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000100:00000040:3.0:1787829677.949343:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829677.949345:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:3.0:1787829677.949346:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829677.949350:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829677.949352:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1787829677.949354:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829677.949358:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829677.949359:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829677.949360:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829677.949364:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829677.949368:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949371:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829677.949374:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829677.949377:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949378:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787829677.949379:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829677.949385:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787829677.949385:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829677.949388:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829677.949393:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829677.949396:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949399:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829677.949400:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829677.949401:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829677.949404:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949405:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:3.0:1787829677.949408:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:0.0:1787829677.949411:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829677.949413:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829677.949417:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:3.0:1787829677.949422:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829677.949423:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298498732
00000100:00000010:0.0:1787829677.949430:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf60.
00000100:00000001:3.0:1787829677.949431:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829677.949434:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949438:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787829677.949438:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949447:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298498732
00000100:00000001:3.0:1787829677.949447:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829677.949453:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd20.
00000100:00000040:3.0:1787829677.949453:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3835)
00000100:00000001:0.0:1787829677.949456:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829677.949461:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829677.949463:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949465:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829677.949471:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677]  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829677.949478:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829677.949480:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829677.949481:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829677.949483:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829677.949484:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829677.949486:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829677.949488:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829677.949494:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949498:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829677.949500:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949501:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.949502:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829677.949505:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949509:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829677.949522:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949530:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829677.949532:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949533:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.949534:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949535:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829677.949537:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829677.949539:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829677.949542:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829677.949543:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829677.949546:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829677.949552:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327368064:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829677.949556:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949557:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.949559:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327368064/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949564:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829677.949566:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829677.949568:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829677.949570:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829677.949571:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829677.949572:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829677.949574:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829677.949575:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:0.0:1787829677.949578:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829677.949580:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829677.949581:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:0.0:1787829677.949583:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829677.949584:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949585:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949590:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829677.949591:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949593:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829677.949594:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829677.949595:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829677.949597:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949602:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829677.949603:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949604:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829677.949606:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829677.949607:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829677.949609:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368192:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829677.949611:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829677.949612:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829677.949614:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949615:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829677.949617:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829677.949618:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829677.949621:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829677.949623:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829677.949625:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829677.949627:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368192, portal 10
00000100:00000001:0.0:1787829677.949629:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829677.949629:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00000040:0.0:1787829677.949634:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949639:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829677.949640:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829677.949642:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829677.949643:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327368192, offset 0
00000400:00000200:0.0:1787829677.949646:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829677.949648:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829677.949651:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829677.949653:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829677.949677:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829677.949694:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829677.949697:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829677.949699:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40648:40648:256:4294967295] 192.168.203.116@tcp LPNI seq info [40648:40648:8:4294967295]
00000400:00000200:0.0:1787829677.949703:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829677.949708:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829677.949711:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829677.949714:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829677.949716:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829677.949718:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829677.949721:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829677.949724:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829677.949728:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829677.949730:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829677.949733:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829677.949737:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829677.949740:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829677.949742:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829677.949744:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829677.949748:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949757:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.949760:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949768:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949770:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829677.949773:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829677.949777:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829677.949779:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829677.949781:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298498059
00000100:00000001:0.0:1787829677.949783:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298498732
00000100:00000001:0.0:1787829677.949785:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298498059
00000100:00000001:0.0:1787829677.949786:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298498732
00000100:00000001:0.0:1787829677.949788:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829677.949789:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949790:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829677.949793:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677]  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829677.949798:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829677.949799:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829677.949801:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829677.949802:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829677.949804:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829677.949805:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829677.949807:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829677.949813:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949817:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829677.949818:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949819:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.949820:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829677.949822:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949826:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829677.949828:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949832:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829677.949834:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949835:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.949835:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.949837:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829677.949838:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829677.949840:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829677.949842:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829677.949843:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829677.949846:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829677.949851:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368192:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829677.949853:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829677.949855:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949864:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829677.949865:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949867:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829677.949868:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829677.949869:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829677.949871:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368320:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829677.949873:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829677.949874:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829677.949875:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.949876:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829677.949879:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829677.949880:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829677.949882:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829677.949884:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829677.949886:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829677.949888:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368320, portal 4
00000100:00000001:0.0:1787829677.949890:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829677.949891:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:0.0:1787829677.949894:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949899:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829677.949900:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829677.949902:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829677.949903:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327368320, offset 0
00000400:00000200:0.0:1787829677.949906:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829677.949908:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829677.949910:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829677.949912:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829677.949914:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829677.949916:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829677.949917:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829677.949920:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40649:40649:256:4294967295] 192.168.203.116@tcp LPNI seq info [40649:40649:8:4294967295]
00000400:00000200:0.0:1787829677.949923:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829677.949927:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829677.949930:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829677.949932:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829677.949935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829677.949937:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829677.949939:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829677.949941:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829677.949944:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829677.949946:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829677.949949:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829677.949952:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829677.949955:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829677.949957:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829677.949959:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829677.949963:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949970:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.949974:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.949981:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.949984:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829677.949987:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829677.949989:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829677.949993:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829677.949995:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298498066
00000100:00000001:0.0:1787829677.949997:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298498732
00000100:00000001:0.0:1787829677.949998:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298498066
00000100:00000001:0.0:1787829677.950000:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298498732
00000100:00000001:0.0:1787829677.950001:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829677.950003:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.950004:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829677.950007:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677]  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829677.950013:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829677.950014:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829677.950015:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829677.950017:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829677.950018:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829677.950019:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829677.950021:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829677.950026:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950031:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829677.950032:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950034:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.950035:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829677.950039:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950045:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829677.950049:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950057:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829677.950060:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950062:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.950063:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950065:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829677.950067:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829677.950070:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829677.950074:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829677.950076:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829677.950081:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829677.950087:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368320:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829677.950090:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829677.950092:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950097:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829677.950098:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.950099:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829677.950101:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829677.950102:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829677.950104:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368448:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829677.950106:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829677.950107:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829677.950108:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829677.950109:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829677.950112:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829677.950113:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829677.950115:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829677.950117:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829677.950119:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829677.950121:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368448, portal 4
00000100:00000001:0.0:1787829677.950122:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829677.950123:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:0.0:1787829677.950127:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950131:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829677.950133:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829677.950134:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829677.950136:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327368448, offset 0
00000400:00000200:0.0:1787829677.950139:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829677.950141:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829677.950144:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829677.950146:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829677.950148:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829677.950149:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829677.950151:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829677.950153:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40650:40650:256:4294967295] 192.168.203.116@tcp LPNI seq info [40650:40650:8:4294967295]
00000400:00000200:0.0:1787829677.950156:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829677.950160:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829677.950162:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829677.950164:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829677.950167:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829677.950168:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829677.950171:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829677.950173:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829677.950175:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829677.950177:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829677.950179:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829677.950181:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829677.950183:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829677.950184:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829677.950186:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829677.950188:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950193:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.950195:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950199:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.950200:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829677.950202:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829677.950204:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829677.950206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829677.950208:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298498071
00000100:00000001:0.0:1787829677.950209:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298498732
00000100:00000001:0.0:1787829677.950211:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298498071
00000100:00000001:0.0:1787829677.950212:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298498732
00000100:00000001:0.0:1787829677.950213:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829677.950214:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.950216:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829677.950218:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677]  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829677.950222:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829677.950223:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829677.950224:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829677.950226:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829677.950227:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829677.950228:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829677.950230:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829677.950235:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950239:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829677.950240:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950241:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.950242:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829677.950244:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950248:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829677.950250:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950255:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829677.950257:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950258:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829677.950259:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950260:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829677.950261:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829677.950262:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829677.950264:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829677.950265:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829677.950268:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829677.950272:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368448:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829677.950274:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950276:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.950277:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327368192/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950281:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829677.950282:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829677.950284:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829677.950285:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829677.950286:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829677.950287:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829677.950288:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829677.950289:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829677.950291:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829677.950292:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829677.950293:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:0.0:1787829677.950295:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829677.950296:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950297:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.950299:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327368320/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950303:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829677.950304:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829677.950305:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829677.950307:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829677.950307:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829677.950308:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829677.950309:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829677.950310:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829677.950311:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829677.950312:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829677.950313:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829677.950314:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829677.950315:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950315:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829677.950317:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327368448/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829677.950321:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829677.950322:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829677.950322:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829677.950323:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829677.950324:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829677.950325:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829677.950326:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829677.950327:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829677.950328:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829677.950328:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829677.950329:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:0.0:1787829677.950332:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829677.950332:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950333:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829677.950336:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829677.950337:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829677.950338:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829677.950339:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829678.460765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829678.460771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829678.460772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829678.460776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829678.460779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829678.460782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829678.460784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829678.460786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829678.460788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829678.652753:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298498732, at 4298498752
00000100:00000040:3.0:1787829678.652754:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298498732, at 4298498752
00000100:00000010:2.0:1787829678.652768:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000100:00000010:3.0:1787829678.652771:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00010000:00000001:0.0:1787829678.780789:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829678.780796:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829678.780801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829678.780802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829678.780805:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829678.780807:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829678.780855:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829678.780858:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829678.780861:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829679.484777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829679.484784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829679.484787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829679.484793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829679.484796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829679.484800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829679.484803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829679.484805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829679.484808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829679.804745:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829679.804752:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829679.804757:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829679.804759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829679.804761:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829679.804765:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829679.804767:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829679.804770:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829679.804773:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829679.804775:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829679.804777:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829679.804780:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829679.804833:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829679.804836:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829679.804839:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829680.508743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829680.508748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829680.508750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829680.508754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829680.508757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829680.508766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829680.508768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829680.508770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829680.508773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829680.828730:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829680.828735:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829680.828739:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829680.828740:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829680.828742:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829680.828743:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829680.828759:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829680.828776:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829680.828778:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829681.532770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829681.532776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829681.532777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829681.532781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829681.532784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829681.532787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829681.532789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829681.532791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829681.532793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829681.852796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829681.852803:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829681.852808:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829681.852809:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829681.852812:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829681.852813:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829681.852918:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829681.852922:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829681.852926:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829682.556755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829682.556761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829682.556763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829682.556767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829682.556771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829682.556774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829682.556775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829682.556777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829682.556780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829682.876762:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829682.876768:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829682.876773:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829682.876774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829682.876777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829682.876779:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829682.876828:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829682.876832:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829682.876835:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829683.068771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829683.068783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829683.068786:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829683.068790:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829683.068795:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829683.068804:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3830
00000400:00000200:3.0:1787829683.068813:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829683.068820:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829683.068824:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829683.068828:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.068832:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.068834:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.068837:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829683.068842:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829683.068845:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829683.068848:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829683.068852:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829683.068856:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829683.068859:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829683.068863:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.068866:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.068868:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.068871:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829683.068875:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829683.068879:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829683.068883:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829683.068886:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829683.068890:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829683.068894:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.068896:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829683.068900:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829683.068910:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829683.068914:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829683.068919:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829683.068921:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.068924:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829683.068928:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829683.068932:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829683.068941:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829683.068947:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829683.068959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829683.068972:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829683.068977:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829683.068983:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829683.069010:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829683.069014:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829683.069019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829683.069022:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829683.069024:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829683.069028:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829683.069031:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3830
00000400:00000200:3.0:1787829683.069035:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829683.069039:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829683.069041:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829683.069045:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069048:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.069050:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069052:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829683.069055:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829683.069057:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829683.069059:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829683.069062:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829683.069065:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829683.069068:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829683.069072:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069074:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.069078:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069080:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829683.069083:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1787829683.069083:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829683.069088:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787829683.069090:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829683.069091:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829683.069097:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829683.069101:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829683.069102:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829683.069110:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829683.069111:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829683.069114:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829683.069116:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.069118:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787829683.069120:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829683.069122:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829683.069126:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787829683.069127:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1787829683.069129:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327368576:192.168.203.116@tcp:250:kworker.0
00000008:00000001:3.0:1787829683.069134:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069135:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829683.069139:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.069140:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829683.069144:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.069146:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1787829683.069147:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
02000000:00000010:0.0:1787829683.069152:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
00000020:00000040:3.0:1787829683.069152:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829683.069156:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829683.069159:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829683.069163:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829683.069163:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.069168:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1787829683.069169:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
02000000:00000001:3.0:1787829683.069171:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1787829683.069176:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
02000000:00000001:3.0:1787829683.069177:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829683.069185:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368576, portal 25
02000000:00000010:3.0:1787829683.069186:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:0.0:1787829683.069188:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829683.069191:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000001:3.0:1787829683.069194:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829683.069197:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787829683.069205:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829683.069207:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829683.069213:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829683.069216:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829683.069222:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327368576, offset 0
00000100:00000001:3.0:1787829683.069224:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829683.069229:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829683.069230:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829683.069239:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829683.069239:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829683.069248:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829683.069257:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787829683.069259:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829683.069262:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:3.0:1787829683.069265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829683.069266:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829683.069272:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:3.0:1787829683.069274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829683.069278:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829683.069280:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40651:40651:256:4294967295] 192.168.203.116@tcp LPNI seq info [40651:40651:8:4294967295]
00000100:00080000:3.0:1787829683.069281:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829683.069287:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829683.069293:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829683.069294:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3830
00000400:00000200:0.0:1787829683.069310:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787829683.069320:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.0:1787829683.069322:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787829683.069327:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:0.0:1787829683.069331:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00080000:3.0:1787829683.069332:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829683.069338:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069343:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1787829683.069344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829683.069349:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829683.069351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:3.0:1787829683.069354:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829683.069360:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:3.0:1787829683.069361:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.1:1787829683.069363:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000020:00000040:3.0:1787829683.069366:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000100:0.0:1787829683.069369:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829683.069370:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:0.0:1787829683.069372:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000020:00000040:3.0:1787829683.069375:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829683.069378:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:3.0:1787829683.069380:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829683.069385:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1787829683.069386:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829683.069390:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000100:00000040:3.0:1787829683.069394:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829683.069395:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829683.069401:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787829683.069401:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.069406:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829683.069408:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829683.069412:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829683.069418:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:0.0:1787829683.069421:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1787829683.069426:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:0.0:1787829683.069427:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829683.069433:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829683.069438:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829683.069443:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829683.069444:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829683.069448:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1787829683.069450:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.069454:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829683.069457:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787829683.069457:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829683.069464:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829683.069468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829683.069469:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069475:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298503852
00000100:00000010:3.0:1787829683.069477:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829683.069481:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000010:0.0:1787829683.069484:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc660.
02000000:00000001:3.0:1787829683.069485:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829683.069488:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.069492:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829683.069493:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829683.069494:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:3.0:1787829683.069496:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069500:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298503852
02000000:00000010:3.0:1787829683.069504:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000010:0.0:1787829683.069506:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccde0.
00000100:00000001:0.0:1787829683.069511:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787829683.069511:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069516:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829683.069519:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829683.069522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829683.069523:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829683.069529:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829683/real 1787829683]  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829683.069544:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:0.0:1787829683.069547:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787829683.069549:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829683.069550:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829683.069553:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829683.069556:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829683.069560:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000040:3.0:1787829683.069561:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.069562:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829683.069569:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829683.069591:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829683.069594:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829683.069596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829683.069606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829683.069610:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829683.069613:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:3.0:1787829683.069614:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829683.069617:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829683.069619:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829683.069622:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:3.0:1787829683.069626:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3830
00000100:00000001:0.0:1787829683.069627:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787829683.069635:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:0.0:1787829683.069637:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787829683.069642:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829683.069680:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829683.069685:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069688:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.069691:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069694:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829683.069697:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829683.069699:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829683.069701:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829683.069704:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829683.069707:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829683.069710:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829683.069713:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069715:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829683.069719:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069725:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829683.069730:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829683.069732:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787829683.069736:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069741:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829683.069743:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829683.069745:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829683.069750:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829683.069756:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.069761:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829683.069762:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00100000:3.0:1787829683.069765:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829683.069768:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:3.0:1787829683.069770:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069772:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787829683.069775:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069777:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.069782:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000010:3.0:1787829683.069783:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829683.069784:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:3.0:1787829683.069789:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00080000:0.0:1787829683.069790:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1787829683.069797:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:0.0:1787829683.069798:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829683.069802:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:3.0:1787829683.069802:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.069808:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829683.069811:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829683.069812:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829683.069817:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829683.069824:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327368576:192.168.203.116@tcp:250:kworker.0
02000000:00000010:3.0:1787829683.069824:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:0.0:1787829683.069831:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829683.069832:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069834:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.069839:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327368576/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787829683.069844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829683.069847:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829683.069850:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829683.069853:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829683.069858:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829683.069861:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829683.069863:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000100:00000001:3.0:1787829683.069863:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000020:00000001:0.0:1787829683.069867:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00000001:3.0:1787829683.069868:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829683.069870:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:0.0:1787829683.069875:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829683.069877:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829683.069879:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000040:3.0:1787829683.069880:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.069882:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829683.069884:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.069889:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.069896:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829683.069899:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829683.069899:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.069905:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829683.069908:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787829683.069909:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3840)
00000100:00000001:0.0:1787829683.069910:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829683.069915:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.069921:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829683.069922:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.069924:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829683.069925:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829683.069927:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829683.069930:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368704:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829683.069932:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829683.069934:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829683.069935:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.069937:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829683.069939:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829683.069941:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829683.069944:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829683.069946:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829683.069949:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829683.069951:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368704, portal 10
00000100:00000001:0.0:1787829683.069953:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829683.069954:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:0.0:1787829683.069958:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.069963:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829683.069965:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829683.069966:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829683.069968:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327368704, offset 0
00000400:00000200:0.0:1787829683.069971:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829683.069973:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829683.069976:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829683.069979:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829683.069981:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829683.069982:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829683.069984:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829683.069986:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40652:40652:256:4294967295] 192.168.203.116@tcp LPNI seq info [40652:40652:8:4294967295]
00000400:00000200:0.0:1787829683.069989:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829683.069994:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829683.069998:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829683.070000:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829683.070003:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829683.070005:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829683.070008:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829683.070009:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829683.070012:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829683.070013:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829683.070016:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829683.070019:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829683.070022:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829683.070024:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829683.070027:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829683.070031:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070040:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.070043:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070050:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070052:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829683.070055:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829683.070059:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829683.070062:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829683.070065:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298503179
00000100:00000001:0.0:1787829683.070068:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298503852
00000100:00000001:0.0:1787829683.070070:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298503179
00000100:00000001:0.0:1787829683.070073:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298503852
00000100:00000001:0.0:1787829683.070075:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829683.070078:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070080:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829683.070084:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829683/real 1787829683]  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829683.070093:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829683.070095:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829683.070097:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829683.070099:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829683.070100:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829683.070102:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829683.070104:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829683.070111:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070115:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829683.070116:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070118:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829683.070119:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829683.070121:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070125:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829683.070127:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070132:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829683.070134:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070135:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829683.070136:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070137:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829683.070139:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829683.070140:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829683.070143:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829683.070144:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829683.070147:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829683.070152:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368704:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829683.070155:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829683.070157:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070161:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829683.070162:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.070163:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829683.070165:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829683.070166:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829683.070168:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368832:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829683.070170:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829683.070171:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829683.070172:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.070174:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829683.070176:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829683.070178:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829683.070180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829683.070182:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829683.070184:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829683.070186:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368832, portal 4
00000100:00000001:0.0:1787829683.070187:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829683.070188:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829683.070192:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070197:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829683.070199:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829683.070200:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829683.070202:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327368832, offset 0
00000400:00000200:0.0:1787829683.070205:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829683.070207:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829683.070210:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829683.070212:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829683.070214:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829683.070216:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829683.070217:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829683.070220:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40653:40653:256:4294967295] 192.168.203.116@tcp LPNI seq info [40653:40653:8:4294967295]
00000400:00000200:0.0:1787829683.070222:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829683.070227:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829683.070230:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829683.070232:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829683.070235:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829683.070237:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829683.070240:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829683.070241:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829683.070244:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829683.070245:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829683.070248:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829683.070250:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829683.070252:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829683.070253:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829683.070255:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829683.070257:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070262:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.070265:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070269:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070271:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829683.070273:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829683.070275:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829683.070278:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829683.070281:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298503186
00000100:00000001:0.0:1787829683.070283:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298503852
00000100:00000001:0.0:1787829683.070285:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298503186
00000100:00000001:0.0:1787829683.070288:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298503852
00000100:00000001:0.0:1787829683.070290:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829683.070292:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070294:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829683.070298:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829683/real 1787829683]  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829683.070307:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829683.070308:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829683.070310:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829683.070313:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829683.070315:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829683.070318:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829683.070322:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829683.070332:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070337:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829683.070338:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070339:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829683.070340:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829683.070343:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070347:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829683.070350:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070355:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829683.070357:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070358:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829683.070359:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070360:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829683.070361:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829683.070363:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829683.070365:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829683.070366:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829683.070368:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829683.070373:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368832:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829683.070376:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829683.070378:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070382:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829683.070383:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.070384:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829683.070385:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829683.070387:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829683.070388:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368960:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829683.070390:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829683.070392:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829683.070393:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829683.070394:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829683.070396:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829683.070398:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829683.070400:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829683.070402:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829683.070404:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829683.070407:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327368960, portal 4
00000100:00000001:0.0:1787829683.070408:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829683.070409:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:0.0:1787829683.070413:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070418:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829683.070419:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829683.070421:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000200:0.0:1787829683.070422:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327368960, offset 0
00000400:00000200:0.0:1787829683.070425:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829683.070427:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829683.070430:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829683.070432:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829683.070435:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829683.070438:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829683.070441:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829683.070444:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40654:40654:256:4294967295] 192.168.203.116@tcp LPNI seq info [40654:40654:8:4294967295]
00000400:00000200:0.0:1787829683.070448:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829683.070455:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829683.070460:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829683.070464:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829683.070468:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829683.070471:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829683.070476:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829683.070479:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829683.070483:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829683.070485:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829683.070488:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829683.070491:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829683.070493:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:0.0:1787829683.070494:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:0.0:1787829683.070497:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829683.070500:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070505:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.070507:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829708 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070512:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070513:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829683.070515:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829683.070517:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829683.070519:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829683.070521:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298503191
00000100:00000001:0.0:1787829683.070522:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298503852
00000100:00000001:0.0:1787829683.070524:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298503191
00000100:00000001:0.0:1787829683.070525:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298503852
00000100:00000001:0.0:1787829683.070527:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829683.070528:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070529:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829683.070532:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829683/real 1787829683]  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829683.070536:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829683.070537:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829683.070538:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829683.070540:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:0.0:1787829683.070541:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:0.0:1787829683.070542:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829683.070544:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829683.070550:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070558:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829683.070559:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070561:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829683.070562:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829683.070564:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070568:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829683.070570:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070574:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829683.070576:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070577:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829683.070578:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070579:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829683.070580:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829683.070582:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829683.070583:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829683.070585:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829683.070587:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829683.070593:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327368960:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829683.070595:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070596:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.070598:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327368704/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070602:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829683.070603:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829683.070605:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829683.070607:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829683.070608:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829683.070609:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829683.070610:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829683.070612:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829683.070614:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829683.070615:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829683.070616:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829683.070618:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829683.070619:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070620:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.070622:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327368832/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070627:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829683.070627:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829683.070628:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829683.070630:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829683.070630:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829683.070631:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829683.070632:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829683.070633:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829683.070634:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829683.070635:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829683.070636:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829683.070637:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829683.070638:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070639:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829683.070641:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327368960/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829708 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829683.070677:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829683.070678:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829683.070679:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829683.070681:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829683.070682:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829683.070682:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829683.070684:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829683.070685:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829683.070687:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829683.070688:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829683.070689:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829683.070690:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829683.070691:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070693:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829683.070695:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829683.070696:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829683.070698:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829683.070699:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829683.580760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829683.580765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829683.580767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829683.580770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829683.580773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829683.580776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829683.580778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829683.580780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829683.580782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829683.772788:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298503852, at 4298503872
00000100:00000040:2.0:1787829683.772792:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298503852, at 4298503872
00000100:00000010:2.0:1787829683.772805:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00000100:00000010:3.0:1787829683.772808:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00010000:00000001:0.0:1787829683.900763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829683.900771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829683.900777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829683.900779:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829683.900783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829683.900784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829683.900804:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829683.900807:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829683.900810:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829684.604743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829684.604748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829684.604750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829684.604754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829684.604757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829684.604760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829684.604762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829684.604764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829684.604766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924819:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829684.924828:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829684.924831:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829684.924836:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924842:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829684.924845:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924847:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924850:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924853:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829684.924855:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829684.924857:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829684.924859:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924860:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829684.924862:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924863:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924865:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829684.924868:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829684.924871:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829684.924995:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829684.925002:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829684.925036:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829685.628768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829685.628773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829685.628775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829685.628779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829685.628782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829685.628785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829685.628788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829685.628790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829685.628793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829685.948812:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829685.948818:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829685.948822:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829685.948824:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829685.948827:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829685.948828:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829685.948876:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829685.948880:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829685.948883:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829686.652719:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829686.652724:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829686.652726:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829686.652730:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829686.652732:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829686.652735:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829686.652737:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829686.652739:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829686.652741:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829686.972806:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829686.972813:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829686.972820:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829686.972823:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829686.972826:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829686.972828:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829686.972876:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829686.972880:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829686.972883:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829687.676787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829687.676792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829687.676794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829687.676798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829687.676801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829687.676804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829687.676806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829687.676808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829687.676810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829687.996780:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829687.996788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829687.996794:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829687.996796:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829687.996800:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829687.996802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829687.996881:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829687.996885:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829687.996887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829688.188846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829688.188855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829688.188857:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829688.188860:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829688.188865:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829688.188870:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3835
00000400:00000200:1.0:1787829688.188877:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829688.188882:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829688.188885:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829688.188887:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.188889:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.188891:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.188893:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829688.188895:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829688.188897:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829688.188899:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829688.188902:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829688.188904:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829688.188905:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829688.188907:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.188909:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.188910:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.188912:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829688.188914:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829688.188917:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829688.188918:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829688.188920:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829688.188923:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829688.188925:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.188927:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829688.188928:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829688.188937:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829688.188939:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829688.188942:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829688.188943:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.188946:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829688.188947:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829688.188951:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829688.188958:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829688.188962:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829688.188969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829688.188977:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829688.188980:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829688.188984:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829688.189004:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829688.189006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829688.189008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829688.189010:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829688.189011:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829688.189013:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829688.189015:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3835
00000400:00000200:1.0:1787829688.189018:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829688.189019:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829688.189021:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829688.189023:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189036:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.189037:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189039:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829688.189040:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829688.189041:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829688.189042:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829688.189044:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829688.189045:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829688.189047:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829688.189049:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189050:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.189052:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189054:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829688.189055:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829688.189058:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787829688.189058:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189060:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829688.189064:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829688.189068:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829688.189069:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189071:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829688.189073:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829688.189076:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:1.0:1787829688.189082:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000040:0.0:1787829688.189084:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829688.189086:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189088:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829688.189100:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829688.189102:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829688.189107:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829688.189109:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189112:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829688.189114:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829688.189117:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189122:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000010:1.0:1787829688.189125:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:0.0:1787829688.189126:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189129:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189133:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:1.0:1787829688.189136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829688.189138:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829688.189146:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189147:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829688.189150:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829688.189157:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327369088:192.168.203.116@tcp:250:kworker.0
00000100:00000040:1.0:1787829688.189158:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189171:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829688.189173:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189176:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:1.0:1787829688.189176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829688.189181:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829688.189181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829688.189184:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:0.0:1787829688.189187:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:1.0:1787829688.189188:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829688.189192:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829688.189194:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3835
02000000:00000010:0.0:1787829688.189195:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
00000400:00000200:1.0:1787829688.189199:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:0.0:1787829688.189201:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829688.189204:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829688.189206:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829688.189211:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189212:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829688.189213:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.189218:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189220:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:0.0:1787829688.189221:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000020:00000001:1.0:1787829688.189221:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829688.189223:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829688.189227:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829688.189229:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829688.189230:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829688.189232:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:0.0:1787829688.189233:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000040:1.0:1787829688.189234:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189240:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.189241:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189243:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829688.189244:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000200:0.0:1787829688.189245:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369088, portal 25
00000100:00080000:1.0:1787829688.189247:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189252:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189253:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1787829688.189253:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829688.189257:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829688.189259:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189260:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829688.189261:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829688.189263:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189264:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189266:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000010:1.0:1787829688.189267:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829688.189271:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829688.189275:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829688.189276:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189279:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829688.189280:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829688.189280:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829688.189283:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829688.189288:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829688.189292:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189297:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:1.0:1787829688.189299:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829688.189305:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829688.189311:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef660.
00000100:00000001:1.0:1787829688.189312:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829688.189315:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829688.189318:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327369088, offset 0
00000100:00000040:1.0:1787829688.189321:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829688.189329:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189334:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829688.189337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829688.189339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829688.189341:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829688.189342:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829688.189343:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829688.189347:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829688.189350:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3835
00000400:00000200:0.0:1787829688.189351:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829688.189357:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829688.189361:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829688.189364:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829688.189366:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189369:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40655:40655:256:4294967295] 192.168.203.116@tcp LPNI seq info [40655:40655:8:4294967295]
00000400:00000200:0.0:1787829688.189372:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829688.189377:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829688.189382:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829688.189386:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829688.189388:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787829688.189392:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189396:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.189400:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829688.189401:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189403:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829688.189406:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:0.0:1787829688.189407:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000020:00000040:1.0:1787829688.189408:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829688.189412:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829688.189416:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:0.0:1787829688.189420:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829688.189421:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000200:0.1:1787829688.189426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829688.189427:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829688.189434:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829688.189437:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829688.189439:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829688.189442:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189443:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829688.189449:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:0.0:1787829688.189450:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829688.189455:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298508972
00000100:00080000:1.0:1787829688.189457:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000010:0.0:1787829688.189461:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc780.
00000100:00000001:1.0:1787829688.189462:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829688.189467:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829688.189470:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829688.189471:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:1.0:1787829688.189472:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189477:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829688.189480:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829688.189481:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298508972
00000008:00000001:1.0:1787829688.189482:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189486:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829688.189487:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc6c0.
00000100:00000010:1.0:1787829688.189490:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829688.189492:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000040:1.0:1787829688.189494:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829688.189498:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829688.189499:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829688.189500:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829688.189501:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829688.189502:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189506:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:1.0:1787829688.189506:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829688.189509:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189510:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189515:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829688.189516:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829688.189521:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829688.189529:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829688.189531:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189534:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000040:1.0:1787829688.189535:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189538:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829688.189543:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829688.189549:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829688.189552:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829688.189553:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829688.189558:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829688.189559:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3845)
00000100:00000001:0.0:1787829688.189568:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829688.189569:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189571:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829688.189573:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829688.189580:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829688.189585:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369216:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829688.189588:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829688.189589:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829688.189591:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189592:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829688.189596:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829688.189598:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829688.189601:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829688.189603:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708d68.
00000400:00000010:0.0:1787829688.189606:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829688.189622:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369216, portal 10
00000100:00000001:0.0:1787829688.189624:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829688.189626:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829688.189632:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189637:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829688.189638:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829688.189640:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef000.
00000100:00000200:0.0:1787829688.189642:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327369216, offset 0
00000400:00000200:0.0:1787829688.189645:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189648:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829688.189651:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829688.189685:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829688.189688:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829688.189690:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829688.189692:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189694:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40656:40656:255:4294967295] 192.168.203.116@tcp LPNI seq info [40656:40656:7:4294967295]
00000400:00000200:0.0:1787829688.189697:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829688.189701:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829688.189704:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829688.189707:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829688.189710:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829688.189712:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829688.189714:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829688.189717:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829688.189719:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829688.189721:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298508299
00000100:00000001:0.0:1787829688.189723:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298508972
00000100:00000001:0.0:1787829688.189724:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298508299
00000100:00000001:0.0:1787829688.189726:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298508972
00000100:00000001:0.0:1787829688.189727:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829688.189728:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189730:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829688.189733:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189739:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829688.189740:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189741:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829688.189743:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829688.189744:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829688.189746:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369344:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829688.189748:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829688.189749:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829688.189750:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189751:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829688.189753:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829688.189755:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829688.189757:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829688.189759:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829688.189761:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829688.189763:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369344, portal 4
00000100:00000001:0.0:1787829688.189764:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829688.189765:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:0.0:1787829688.189769:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189774:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829688.189775:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829688.189777:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fefe58.
00000100:00000200:0.0:1787829688.189778:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327369344, offset 0
00000400:00000200:0.0:1787829688.189781:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189783:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829688.189785:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829688.189787:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829688.189788:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829688.189790:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829688.189791:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189793:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40657:40657:254:4294967295] 192.168.203.116@tcp LPNI seq info [40657:40657:6:4294967295]
00000400:00000200:0.0:1787829688.189796:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829688.189800:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829688.189803:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829688.189806:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7000.
00000800:00000200:0.0:1787829688.189808:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829688.189810:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829688.189812:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829688.189813:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829688.189824:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829688.189826:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298508306
00000100:00000001:0.0:1787829688.189830:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298508972
00000100:00000001:0.0:1787829688.189832:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298508306
00000100:00000001:0.0:1787829688.189833:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298508972
00000100:00000001:0.0:1787829688.189835:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829688.189836:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189837:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189839:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829688.189841:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829688.189842:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829688.189843:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829688.189844:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829688.189846:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829688.189849:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189853:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829688.189854:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189856:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829688.189857:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829688.189858:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829688.189860:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369472:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829688.189863:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829688.189865:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829688.189866:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829688.189867:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829688.189869:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829688.189870:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829688.189872:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829688.189873:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829688.189876:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef330.
00000100:00000200:0.0:1787829688.189878:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369472, portal 4
00000100:00000001:0.0:1787829688.189879:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829688.189880:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829688.189884:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829688.189888:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829688.189890:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829688.189892:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fefd48.
00000100:00000200:0.0:1787829688.189893:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327369472, offset 0
00000400:00000200:0.0:1787829688.189896:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189898:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829688.189900:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829688.189901:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829688.189903:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829688.189904:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829688.189906:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829688.189909:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40658:40658:253:4294967295] 192.168.203.116@tcp LPNI seq info [40658:40658:5:4294967295]
00000400:00000200:0.0:1787829688.189913:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829688.189919:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829688.189924:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829688.189927:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7f00.
00000800:00000200:0.0:1787829688.189931:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829688.189935:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829688.189938:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829688.189941:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829688.189944:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829688.189946:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298508311
00000100:00000001:0.0:1787829688.189949:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298508972
00000100:00000001:0.0:1787829688.189951:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298508311
00000100:00000001:0.0:1787829688.189953:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298508972
00000100:00000001:0.0:1787829688.189955:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829688.189957:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189959:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189961:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829688.189964:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829688.189966:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829688.189968:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829688.189969:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829688.189971:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829688.189973:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829688.700784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829688.700793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829688.700795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829688.700801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829688.700805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829688.700809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829688.700813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829688.700816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829688.700820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829688.892707:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298508972, at 4298508992
00000100:00000010:3.0:1787829688.892724:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000100:00000040:2.0:1787829688.892734:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298508972, at 4298508992
00000100:00000010:2.0:1787829688.892747:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00010000:00000001:0.0:1787829689.020794:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829689.020799:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829689.020804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829689.020805:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829689.020808:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829689.020810:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829689.020870:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829689.020877:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829689.020881:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829689.724721:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829689.724726:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829689.724729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829689.724733:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829689.724737:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829689.724740:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829689.724742:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829689.724744:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829689.724747:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829690.044798:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829690.044808:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829690.044816:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829690.044817:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829690.044822:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829690.044842:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829690.044845:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829690.044850:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829690.044854:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829690.044857:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829690.044859:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829690.044862:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829690.044904:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829690.044907:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829690.044909:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829690.748790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829690.748798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829690.748802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829690.748810:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829690.748826:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829690.748832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829690.748836:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829690.748840:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829690.748845:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829691.068755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829691.068760:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829691.068763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829691.068764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829691.068766:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829691.068767:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829691.068816:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829691.068820:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829691.068823:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829691.261021:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829691.261101:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829691.261110:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829691.261114:0:2336:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829691.261120:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829691.261124:0:2336:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829691.261129:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829691.261133:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:2.0:1787829691.261156:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829691.261164:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829691.261174:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef660
00000400:00000010:2.0:1787829691.261180:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef660.
00000100:00000001:2.0:1787829691.261187:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829691.261201:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829691.261259:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829691.261267:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829691.261280:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829691.261283:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829691.261295:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829691.261300:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:2.0:1787829691.261305:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829691.261312:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829691.261317:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef000
00000400:00000010:2.0:1787829691.261319:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef000.
00000100:00000001:2.0:1787829691.261324:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829691.261328:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000200:2.0:1787829691.261331:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.261337:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829691.261344:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829691.261350:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1787829691.261362:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829688/real 1787829691]  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829691.261368:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829691.261401:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829691.261404:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1787829691.261405:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000001:0.0:1787829691.261411:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829691.261417:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000800:00000100:2.0:1787829691.261421:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000400:00000200:0.0:1787829691.261424:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000800:00000010:2.0:1787829691.261427:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000010:0.0:1787829691.261429:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000400:00000200:2.0:1787829691.261438:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:0.0:1787829691.261439:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1787829691.261453:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000200:0.0:1787829691.261454:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829691.261460:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefe58
00000400:00000010:2.0:1787829691.261465:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefe58.
00000100:00000001:2.0:1787829691.261473:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829691.261485:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829691.261486:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.261522:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829691.261527:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.261536:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829691.261540:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787829691.261541:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829691.261549:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829691.261556:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829691.261571:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829691.261577:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829691.261583:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:2.0:1787829691.261611:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000100:00000040:0.0:1787829691.261617:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1787829691.261619:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000200:2.0:1787829691.261627:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829691.261637:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829691.261649:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefd48
00000400:00000010:2.0:1787829691.261654:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefd48.
00000100:00000001:0.0:1787829691.261715:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829691.261721:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.261724:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.261726:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.261730:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829691.261734:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829691.261739:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1787829691.261750:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1787829691.261751:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829691.261759:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000200:2.0:1787829691.261763:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829691.261771:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829691.261790:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829691.261799:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829691.261802:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327369088:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829691.261819:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829691.261823:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829691.261826:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829691.261830:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787829691.261836:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327369088/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.261848:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829691.261852:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829691.261858:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829691.261864:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829691.261868:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829691.261871:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829691.261874:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829691.261877:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:0.0:1787829691.261882:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829691.261885:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829691.261887:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829691.261892:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829691.261894:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.261898:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829691.261902:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829691.261904:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829691.261907:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829691.261913:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829688/real 1787829691]  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829691.261925:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829691.261927:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829691.261930:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829691.261933:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef330
00000400:00000010:0.0:1787829691.261936:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef330.
00000100:00000001:0.0:1787829691.261940:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829691.261945:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829691.261957:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.261967:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829691.261970:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.261973:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.261975:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829691.261982:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.261991:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829691.261997:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262006:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829691.262010:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262012:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.262013:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262016:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829691.262018:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829691.262022:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829691.262028:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829691.262031:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829691.262037:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829691.262050:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369472:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829691.262056:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829691.262080:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829688/real 1787829691]  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829691.262093:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829691.262096:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829691.262099:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708d68.
00000400:00000200:0.0:1787829691.262103:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829691.262106:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829691.262109:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829691.262115:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829691.262128:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262138:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829691.262141:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262143:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.262146:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829691.262151:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262161:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829691.262168:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262184:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829691.262187:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262190:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.262192:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262195:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829691.262198:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829691.262201:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829691.262206:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829691.262209:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829691.262215:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829691.262227:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369216:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829691.262233:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829691.262238:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829688/real 1787829691]  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829691.262250:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829691.262252:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829691.262254:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829691.262258:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829691.262260:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829691.262263:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829691.262268:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829691.262279:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262287:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829691.262289:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262292:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.262294:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829691.262299:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262309:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829691.262315:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262326:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829691.262329:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262332:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829691.262333:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262336:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829691.262338:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829691.262341:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829691.262346:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829691.262349:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829691.262355:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829691.262366:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369344:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829691.262372:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262374:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829691.262379:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327369472/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262388:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829691.262391:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829691.262394:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829691.262399:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829691.262401:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829691.262403:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829691.262406:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829691.262408:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:0.0:1787829691.262411:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829691.262413:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829691.262415:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829691.262419:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829691.262421:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262424:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829691.262429:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327369216/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262439:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829691.262442:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829691.262444:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829691.262448:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829691.262450:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829691.262452:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829691.262454:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829691.262457:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829691.262460:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829691.262462:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829691.262464:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829691.262468:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829691.262470:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262473:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829691.262477:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327369344/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829691.262485:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829691.262487:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829691.262489:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829691.262492:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829691.262494:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829691.262496:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829691.262498:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829691.262500:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829691.262504:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829691.262506:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829691.262508:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829691.262511:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829691.262513:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262515:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829691.262523:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829691.262526:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829691.262529:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829691.262531:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829691.772847:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829691.772854:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829691.772857:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829691.772862:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829691.772867:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829691.772870:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829691.772873:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829691.772876:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829691.772880:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829692.092781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829692.092787:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829692.092792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829692.092793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829692.092796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829692.092798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829692.092857:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829692.092860:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829692.092864:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829692.796735:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829692.796741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829692.796743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829692.796746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829692.796749:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829692.796752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829692.796754:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829692.796756:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829692.796758:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829693.116782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829693.116788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829693.116792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829693.116794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829693.116799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829693.116801:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829693.116838:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829693.116840:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829693.116842:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829693.308773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829693.308781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829693.308783:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829693.308786:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829693.308791:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829693.308798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3840
00000400:00000200:0.0:1787829693.308804:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829693.308809:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829693.308812:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829693.308814:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.308816:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829693.308819:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.308820:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829693.308823:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829693.308825:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829693.308827:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829693.308830:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829693.308831:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829693.308833:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829693.308834:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.308836:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829693.308837:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.308838:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829693.308840:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829693.308842:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829693.308844:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829693.308846:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829693.308849:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829693.308851:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.308853:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829693.308855:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829693.308860:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716da40.
00000020:00000040:0.0:1787829693.308863:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829693.308866:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829693.308867:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.308869:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829693.308871:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829693.308875:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829693.308881:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533800.
00000100:00000001:0.0:1787829693.308886:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829693.308893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829693.308901:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829693.308904:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829693.308908:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000226d20b5] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829693.308929:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829693.308931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829693.308933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829693.308934:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829693.308936:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829693.308939:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829693.308941:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3840
00000400:00000200:0.0:1787829693.308946:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829693.308950:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829693.308953:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829693.308956:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.308959:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787829693.308960:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787829693.308962:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829693.308966:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829693.308967:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:2.0:1787829693.308970:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787829693.308972:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829693.308975:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829693.308977:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829693.308981:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000040:2.0:1787829693.308982:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787829693.308985:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:0.0:1787829693.308991:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829693.308999:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829693.309001:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829693.309006:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.309010:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787829693.309012:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787829693.309014:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787829693.309018:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:2.0:1787829693.309020:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309023:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:2.0:1787829693.309029:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327369600:192.168.203.116@tcp:250:kworker.0
00000020:00000040:0.0:1787829693.309030:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:2.0:1787829693.309038:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1787829693.309039:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:2.0:1787829693.309045:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787829693.309046:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.309050:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829693.309051:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829693.309056:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787829693.309061:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1787829693.309064:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829693.309066:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.309069:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309071:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829693.309077:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787829693.309078:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00000001:0.0:1787829693.309085:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787829693.309086:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000100:00000001:0.0:1787829693.309088:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787829693.309095:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000010:0.0:1787829693.309096:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716c000.
00000020:00000040:0.0:1787829693.309102:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829693.309106:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:2.0:1787829693.309106:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369600, portal 25
02000000:00000001:0.0:1787829693.309110:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309110:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829693.309113:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000001:0.0:1787829693.309115:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829693.309118:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1787829693.309138:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829693.309139:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829693.309148:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532400.
00000100:00000001:2.0:1787829693.309150:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829693.309155:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829693.309156:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787829693.309158:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829693.309163:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327369600, offset 0
00000100:00100000:0.0:1787829693.309168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787829693.309173:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829693.309184:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787829693.309186:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829693.309205:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.309213:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:0.0:1787829693.309215:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cdd0ad0f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829693.309222:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829693.309230:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:0.0:1787829693.309233:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.309237:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1787829693.309238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787829693.309243:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829693.309251:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40659:40659:256:4294967295] 192.168.203.116@tcp LPNI seq info [40659:40659:8:4294967295]
00000100:00080000:0.0:1787829693.309253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1787829693.309258:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829693.309259:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829693.309263:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829693.309267:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829693.309275:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3840
00000400:00000200:2.0:1787829693.309278:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787829693.309285:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:2.0:1787829693.309291:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829693.309295:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:2.0:1787829693.309300:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000100:00080000:0.0:1787829693.309301:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829693.309308:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.309313:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787829693.309314:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:0.0:1787829693.309319:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:2.0:1787829693.309322:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787829693.309323:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829693.309330:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829693.309334:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:2.1:1787829693.309335:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:0.0:1787829693.309338:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829693.309343:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:2.1:1787829693.309343:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787829693.309350:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000100:2.0:1787829693.309355:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:0.0:1787829693.309357:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:2.0:1787829693.309361:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000100:00000040:0.0:1787829693.309365:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.309369:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829693.309372:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829693.309374:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:2.0:1787829693.309376:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1787829693.309377:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:0.0:1787829693.309386:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:2.0:1787829693.309386:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787829693.309393:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.309395:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
02000000:00000001:0.0:1787829693.309396:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1787829693.309400:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
02000000:00000001:0.0:1787829693.309403:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829693.309407:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1787829693.309408:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309412:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1787829693.309414:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787829693.309416:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829693.309423:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309428:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829693.309436:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f0c0.
00000100:00000001:2.0:1787829693.309438:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787829693.309442:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000040:2.0:1787829693.309444:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829693.309448:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829693.309451:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309456:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787829693.309460:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829693.309461:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787829693.309465:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1787829693.309466:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829693.309474:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531400.
00000100:00000001:2.0:1787829693.309475:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309479:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.309485:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829693.309490:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298514092
00000100:00100000:0.0:1787829693.309491:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1787829693.309497:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee95a0.
00000100:00000001:0.0:1787829693.309505:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829693.309508:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829693.309509:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829693.309516:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003ce953a3] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.309520:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298514092
00000100:00000010:2.0:1787829693.309527:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9d80.
00000100:00000001:0.0:1787829693.309534:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309535:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:0.0:1787829693.309542:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829693.309547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829693.309549:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829693.309551:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829693.309553:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787829693.309556:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787829693.309557:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3840
00000100:00000001:2.0:1787829693.309561:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829693.309567:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829693.309567:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787829693.309589:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00100000:2.0:1787829693.309590:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829693/real 1787829693]  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829693.309595:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829693.309605:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000001:2.0:1787829693.309607:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787829693.309609:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:2.0:1787829693.309611:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1787829693.309616:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:2.0:1787829693.309616:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000100:00000001:0.0:1787829693.309623:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:2.0:1787829693.309624:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000020:00000001:0.0:1787829693.309626:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:2.0:1787829693.309629:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000020:00000040:0.0:1787829693.309631:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:2.0:1787829693.309633:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:0.0:1787829693.309635:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829693.309641:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000200:2.0:1787829693.309642:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787829693.309645:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829693.309649:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:2.0:1787829693.309684:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.309692:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:0.0:1787829693.309696:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829693.309697:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829693.309701:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829693.309706:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829693.309707:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829693.309710:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:2.0:1787829693.309711:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1787829693.309717:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00100000:2.0:1787829693.309722:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829693.309726:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829693.309734:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309737:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829693.309739:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829693.309744:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787829693.309745:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829693.309751:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829693.309756:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829693.309761:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787829693.309762:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:0.0:1787829693.309765:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309767:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829693.309770:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309771:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.309775:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1787829693.309777:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716e580.
00000100:00000001:2.0:1787829693.309778:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:0.0:1787829693.309783:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:2.0:1787829693.309783:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829693.309791:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:0.0:1787829693.309792:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829693.309796:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829693.309799:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829693.309801:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829693.309805:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787829693.309805:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:0.0:1787829693.309807:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829693.309813:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829693.309815:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533000.
00000100:00000001:0.0:1787829693.309822:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829693.309830:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327369600:192.168.203.116@tcp:250:kworker.0
00000100:00100000:0.0:1787829693.309834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829693.309837:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.309842:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.309849:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327369600/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829693.309850:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829693.309855:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309865:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000040:0.0:1787829693.309866:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc9f2c41] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829693.309871:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829693.309876:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003000.
00000100:00000001:0.0:1787829693.309880:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.309882:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829693.309885:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000040:0.0:1787829693.309887:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3851)
00000020:00000040:2.0:1787829693.309888:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787829693.309894:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829693.309897:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:2.0:1787829693.309902:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829693.309905:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829693.309907:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:2.0:1787829693.309912:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829693.309914:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.309916:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.309926:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829693.309928:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.309930:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829693.309932:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829693.309934:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829693.309938:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.309945:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829693.309947:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.309949:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829693.309951:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829693.309954:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829693.309957:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369728:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829693.309961:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829693.309963:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829693.309965:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.309967:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829693.309971:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87003000.
02000000:00000001:2.0:1787829693.309973:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.309977:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829693.309981:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829693.309985:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829693.309988:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369728, portal 10
00000100:00000001:2.0:1787829693.309991:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829693.309992:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:2.0:1787829693.310000:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310005:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829693.310008:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829693.310011:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829693.310014:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327369728, offset 0
00000400:00000200:2.0:1787829693.310018:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829693.310024:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829693.310028:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829693.310032:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829693.310035:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829693.310038:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829693.310040:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829693.310043:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40660:40660:256:4294967295] 192.168.203.116@tcp LPNI seq info [40660:40660:8:4294967295]
00000400:00000200:2.0:1787829693.310047:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829693.310054:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829693.310058:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829693.310062:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829693.310066:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829693.310069:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787829693.310074:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829693.310077:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787829693.310080:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829693.310082:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829693.310085:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829693.310088:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829693.310091:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829693.310093:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829693.310095:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829693.310099:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310108:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.310111:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310118:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310120:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829693.310124:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829693.310127:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787829693.310131:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829693.310133:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298513419
00000100:00000001:2.0:1787829693.310136:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298514092
00000100:00000001:2.0:1787829693.310138:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298513419
00000100:00000001:2.0:1787829693.310140:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298514092
00000100:00000001:2.0:1787829693.310143:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829693.310145:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310147:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829693.310152:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829693/real 1787829693]  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829693.310161:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829693.310162:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829693.310164:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829693.310166:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829693.310170:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829693.310172:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829693.310175:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829693.310184:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310191:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829693.310193:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310195:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.310197:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829693.310200:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310207:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829693.310211:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310218:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829693.310221:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310223:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.310225:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310227:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829693.310229:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829693.310232:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829693.310235:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829693.310237:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829693.310242:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829693.310251:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369728:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829693.310255:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829693.310259:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310265:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829693.310266:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.310269:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829693.310271:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829693.310273:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829693.310276:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369856:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829693.310280:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829693.310282:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829693.310284:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.310286:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829693.310290:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787829693.310292:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.310296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829693.310299:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829693.310302:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829693.310306:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369856, portal 4
00000100:00000001:2.0:1787829693.310308:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829693.310310:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:2.0:1787829693.310315:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310322:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829693.310324:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829693.310326:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829693.310329:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327369856, offset 0
00000400:00000200:2.0:1787829693.310334:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829693.310338:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829693.310342:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829693.310346:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829693.310349:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829693.310351:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829693.310353:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829693.310357:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40661:40661:256:4294967295] 192.168.203.116@tcp LPNI seq info [40661:40661:8:4294967295]
00000400:00000200:2.0:1787829693.310361:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829693.310367:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829693.310371:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829693.310373:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829693.310376:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829693.310378:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787829693.310381:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829693.310383:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787829693.310386:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829693.310387:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829693.310390:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829693.310392:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829693.310394:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829693.310395:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829693.310397:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829693.310400:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310405:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.310408:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310412:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310413:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829693.310416:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829693.310418:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787829693.310420:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829693.310421:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298513426
00000100:00000001:2.0:1787829693.310423:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298514092
00000100:00000001:2.0:1787829693.310425:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298513426
00000100:00000001:2.0:1787829693.310426:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298514092
00000100:00000001:2.0:1787829693.310428:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829693.310429:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310430:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829693.310432:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829693/real 1787829693]  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829693.310438:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829693.310439:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829693.310440:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829693.310441:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829693.310443:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829693.310444:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829693.310446:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829693.310451:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310455:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829693.310456:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310457:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.310458:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829693.310461:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310465:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829693.310468:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310472:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829693.310473:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310474:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.310475:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310476:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829693.310477:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829693.310479:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829693.310480:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829693.310482:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829693.310484:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829693.310489:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369856:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829693.310491:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829693.310493:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310497:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829693.310498:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.310500:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829693.310501:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829693.310502:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829693.310504:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369984:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829693.310506:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829693.310508:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829693.310509:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829693.310510:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829693.310512:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829693.310514:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829693.310516:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829693.310517:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829693.310519:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829693.310521:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327369984, portal 4
00000100:00000001:2.0:1787829693.310522:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829693.310523:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:2.0:1787829693.310528:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310532:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829693.310533:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829693.310535:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829693.310536:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327369984, offset 0
00000400:00000200:2.0:1787829693.310539:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829693.310541:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829693.310543:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829693.310545:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829693.310548:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829693.310550:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829693.310553:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829693.310556:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40662:40662:256:4294967295] 192.168.203.116@tcp LPNI seq info [40662:40662:8:4294967295]
00000400:00000200:2.0:1787829693.310560:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829693.310567:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829693.310585:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829693.310589:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829693.310593:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829693.310596:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787829693.310600:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829693.310603:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787829693.310606:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829693.310609:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829693.310612:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829693.310615:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829693.310618:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829693.310620:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829693.310623:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829693.310627:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310635:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.310639:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310643:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310644:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829693.310647:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829693.310649:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787829693.310651:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829693.310652:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298513431
00000100:00000001:2.0:1787829693.310674:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298514092
00000100:00000001:2.0:1787829693.310676:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298513432
00000100:00000001:2.0:1787829693.310678:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298514092
00000100:00000001:2.0:1787829693.310679:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829693.310681:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310683:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829693.310685:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829693/real 1787829693]  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829693.310691:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829693.310692:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829693.310694:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829693.310695:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829693.310696:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829693.310698:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829693.310700:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829693.310705:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310709:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829693.310710:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310711:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.310712:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829693.310716:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310723:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829693.310727:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310734:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829693.310736:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310737:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829693.310739:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310740:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829693.310742:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829693.310745:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829693.310748:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829693.310751:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829693.310755:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829693.310763:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327369984:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829693.310767:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310769:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.310773:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327369728/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310779:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829693.310781:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829693.310783:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87003000.
02000000:00000001:2.0:1787829693.310786:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829693.310788:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829693.310789:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787829693.310792:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829693.310794:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:2.0:1787829693.310797:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829693.310798:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829693.310799:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:2.0:1787829693.310802:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829693.310803:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310804:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.310807:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327369856/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310811:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829693.310812:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829693.310813:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787829693.310815:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829693.310815:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829693.310816:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787829693.310817:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829693.310818:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:2.0:1787829693.310820:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829693.310820:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829693.310821:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:2.0:1787829693.310823:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829693.310823:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310824:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829693.310826:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327369984/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829693.310830:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829693.310831:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829693.310832:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829693.310834:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829693.310834:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829693.310835:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787829693.310837:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829693.310838:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:2.0:1787829693.310839:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829693.310839:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829693.310840:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:2.0:1787829693.310841:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829693.310842:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310843:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.310846:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829693.310847:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.310848:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829693.310850:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829693.820789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829693.820796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829693.820799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829693.820805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829693.820809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829693.820812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.820815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829693.820819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829693.820822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829694.012740:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298514092, at 4298514112
00000100:00000040:2.0:1787829694.012742:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298514092, at 4298514112
00000100:00000010:3.0:1787829694.012757:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9d80.
00000100:00000010:2.0:1787829694.012769:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee95a0.
00010000:00000001:0.0:1787829694.140784:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829694.140790:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829694.140795:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829694.140796:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829694.140799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829694.140801:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829694.140865:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829694.140868:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829694.140871:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829694.844740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829694.844746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829694.844747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829694.844751:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829694.844754:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829694.844757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829694.844759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829694.844761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829694.844763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829695.164778:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829695.164781:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829695.164784:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164789:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829695.164792:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164795:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164798:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829695.164803:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829695.164804:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829695.164805:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164806:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829695.164807:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164808:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164810:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829695.164812:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829695.164813:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829695.164867:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829695.164871:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829695.164874:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829695.868757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829695.868765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829695.868768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829695.868773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829695.868777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829695.868781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829695.868784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829695.868788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829695.868791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829696.188755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829696.188760:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829696.188765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829696.188766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829696.188769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829696.188770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829696.188787:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829696.188789:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829696.188791:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829696.892756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829696.892761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829696.892763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829696.892768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829696.892771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829696.892774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829696.892777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829696.892780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829696.892785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829697.212778:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829697.212784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829697.212788:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829697.212790:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829697.212792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829697.212794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829697.212864:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829697.212869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829697.212873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829697.916761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829697.916767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829697.916769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829697.916773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829697.916776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829697.916779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829697.916781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829697.916783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829697.916785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829698.236786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829698.236794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829698.236800:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829698.236802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829698.236815:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829698.236816:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829698.236865:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829698.236868:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829698.236871:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829698.428786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829698.428795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829698.428797:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829698.428807:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829698.428811:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829698.428817:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3846
00000400:00000200:2.0:1787829698.428824:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829698.428829:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829698.428832:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829698.428834:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.428836:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829698.428838:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.428840:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829698.428842:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829698.428844:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829698.428846:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829698.428848:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829698.428850:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829698.428852:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829698.428854:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.428855:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829698.428857:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.428858:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829698.428860:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829698.428862:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829698.428864:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829698.428866:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829698.428868:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829698.428870:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.428873:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829698.428875:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829698.428883:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829698.428885:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829698.428888:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829698.428889:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.428892:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829698.428894:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829698.428897:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829698.428902:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829698.428908:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829698.428915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829698.428923:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829698.428926:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829698.428930:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829698.428948:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829698.428950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829698.428953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829698.428954:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829698.428955:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829698.428957:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829698.428959:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3846
00000400:00000200:2.0:1787829698.428962:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829698.428965:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829698.428967:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829698.428969:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829698.428971:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829698.428971:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829698.428976:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829698.428977:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829698.428978:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829698.428981:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787829698.428982:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829698.428985:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829698.428986:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829698.428989:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000040:0.0:1787829698.428991:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829698.428991:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829698.428995:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829698.429001:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429005:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829698.429009:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787829698.429009:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829698.429013:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.429013:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829698.429019:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787829698.429019:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829698.429022:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:0.0:1787829698.429024:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829698.429025:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429029:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829698.429029:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1787829698.429035:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327370112:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787829698.429035:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829698.429040:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429042:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787829698.429043:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829698.429047:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1787829698.429048:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829698.429051:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787829698.429052:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429055:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787829698.429055:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829698.429060:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787829698.429061:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000010:0.0:1787829698.429062:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:2.0:1787829698.429063:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829698.429066:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429067:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.429068:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829698.429071:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829698.429083:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829698.429086:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829698.429089:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829698.429094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829698.429095:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829698.429101:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000100:00000001:2.0:1787829698.429105:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829698.429108:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829698.429109:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000040:2.0:1787829698.429112:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829698.429117:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370112, portal 25
00000100:00000001:0.0:1787829698.429122:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829698.429122:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429124:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00080000:2.0:1787829698.429126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829698.429128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829698.429132:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829698.429133:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829698.429137:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:0.0:1787829698.429138:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829698.429141:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3846
00000400:00000200:2.0:1787829698.429149:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829698.429154:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829698.429156:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787829698.429157:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829698.429163:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787829698.429163:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829698.429169:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000001:2.0:1787829698.429169:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829698.429173:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829698.429175:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327370112, offset 0
00000100:00000001:2.0:1787829698.429175:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829698.429180:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829698.429182:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829698.429183:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829698.429184:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829698.429188:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829698.429189:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829698.429193:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829698.429196:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429197:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829698.429198:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429199:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829698.429208:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829698.429211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829698.429212:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429218:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429219:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:2.0:1787829698.429222:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829698.429225:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1787829698.429228:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829698.429230:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:2.0:1787829698.429234:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429235:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829698.429239:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829698.429240:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829698.429245:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40663:40663:256:4294967295] 192.168.203.116@tcp LPNI seq info [40663:40663:8:4294967295]
00000008:00100000:2.0:1787829698.429245:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829698.429251:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.429254:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429260:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:2.0:1787829698.429262:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787829698.429266:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829698.429271:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829698.429273:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429274:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787829698.429275:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829698.429279:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829698.429281:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829698.429284:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000010:2.0:1787829698.429286:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000800:00000010:0.0:1787829698.429290:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00000001:2.0:1787829698.429290:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829698.429298:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829698.429302:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829698.429308:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829698.429310:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829698.429314:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829698.429317:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829698.429321:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829698.429322:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829698.429333:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429338:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829698.429339:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000100:00080000:2.0:1787829698.429340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829698.429345:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829698.429347:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829698.429348:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829698.429349:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787829698.429350:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829698.429362:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3846
00000400:00000200:0.0:1787829698.429368:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829698.429369:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829698.429375:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829698.429377:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829698.429379:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000100:00000040:2.0:1787829698.429380:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429385:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829698.429386:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000040:2.0:1787829698.429387:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429391:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829698.429394:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:2.0:1787829698.429396:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829698.429398:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829698.429402:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:0.0:1787829698.429403:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829698.429404:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829698.429409:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829698.429413:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829698.429420:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829698.429421:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829698.429422:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829698.429428:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829698.429429:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829698.429430:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829698.429435:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829698.429440:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829698.429445:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829698.429448:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829698.429449:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829698.429451:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829698.429452:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787829698.429456:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.429457:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829698.429459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829698.429460:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787829698.429461:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.429465:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:2.0:1787829698.429471:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829698.429472:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298519212
00000020:00000040:2.0:1787829698.429476:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000010:0.0:1787829698.429479:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccea0.
02000000:00000001:2.0:1787829698.429481:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829698.429484:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429487:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787829698.429487:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829698.429493:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829698.429496:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298519212
02000000:00000001:2.0:1787829698.429496:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829698.429501:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc420.
00000100:00000001:0.0:1787829698.429505:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000010:2.0:1787829698.429505:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829698.429512:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429517:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787829698.429519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829698.429521:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429525:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829698.429531:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698]  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829698.429535:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829698.429552:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829698.429560:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1787829698.429561:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829698.429564:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829698.429567:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829698.429573:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00000001:2.0:1787829698.429574:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829698.429578:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000040:2.0:1787829698.429580:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3856)
00000100:00000001:0.0:1787829698.429582:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829698.429588:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829698.429594:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429599:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829698.429600:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429602:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.429603:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829698.429606:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429611:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829698.429614:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429623:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829698.429625:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429627:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.429628:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429629:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829698.429631:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829698.429633:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829698.429635:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829698.429636:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829698.429639:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829698.429645:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327370112:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829698.429649:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429650:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.429652:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327370112/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429674:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829698.429676:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829698.429678:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829698.429680:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829698.429682:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829698.429683:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829698.429685:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829698.429686:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829698.429689:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829698.429691:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829698.429692:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:0.0:1787829698.429694:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829698.429695:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429697:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429702:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829698.429703:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429705:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829698.429706:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829698.429707:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829698.429710:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429715:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829698.429716:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429717:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829698.429719:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829698.429720:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829698.429722:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370240:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829698.429724:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829698.429725:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829698.429726:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429728:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829698.429730:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829698.429732:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429734:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829698.429736:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829698.429739:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829698.429741:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370240, portal 10
00000100:00000001:0.0:1787829698.429742:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829698.429743:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:0.0:1787829698.429748:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429752:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829698.429754:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829698.429756:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829698.429757:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327370240, offset 0
00000400:00000200:0.0:1787829698.429760:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829698.429763:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829698.429765:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829698.429768:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829698.429770:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829698.429772:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829698.429774:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829698.429776:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40664:40664:256:4294967295] 192.168.203.116@tcp LPNI seq info [40664:40664:8:4294967295]
00000400:00000200:0.0:1787829698.429779:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829698.429783:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829698.429786:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829698.429788:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829698.429791:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829698.429793:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829698.429796:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829698.429797:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829698.429800:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829698.429801:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829698.429803:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829698.429806:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829698.429808:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829698.429809:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829698.429810:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829698.429813:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429818:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.429820:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429825:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429825:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829698.429827:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829698.429830:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829698.429832:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829698.429833:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298518539
00000100:00000001:0.0:1787829698.429835:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298519212
00000100:00000001:0.0:1787829698.429837:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298518539
00000100:00000001:0.0:1787829698.429838:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298519212
00000100:00000001:0.0:1787829698.429840:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829698.429841:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.429842:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829698.429844:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698]  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829698.429849:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829698.429850:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829698.429851:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829698.429852:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829698.429853:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829698.429854:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829698.429856:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829698.429861:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429865:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829698.429866:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429868:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.429869:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829698.429871:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429875:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829698.429877:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429881:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829698.429883:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429884:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.429885:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.429886:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829698.429888:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829698.429890:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829698.429892:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829698.429893:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829698.429895:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829698.429900:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370240:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829698.429902:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829698.429904:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429908:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829698.429909:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429910:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829698.429911:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829698.429912:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829698.429914:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370368:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829698.429916:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829698.429917:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829698.429918:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.429919:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829698.429922:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829698.429923:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.429925:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829698.429927:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829698.429930:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829698.429931:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370368, portal 4
00000100:00000001:0.0:1787829698.429933:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829698.429934:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:0.0:1787829698.429937:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429941:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829698.429943:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829698.429944:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829698.429946:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327370368, offset 0
00000400:00000200:0.0:1787829698.429948:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829698.429950:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829698.429952:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829698.429953:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829698.429955:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829698.429957:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829698.429958:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829698.429960:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40665:40665:256:4294967295] 192.168.203.116@tcp LPNI seq info [40665:40665:8:4294967295]
00000400:00000200:0.0:1787829698.429963:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829698.429967:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829698.429969:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829698.429971:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829698.429973:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829698.429975:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829698.429978:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829698.429979:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829698.429982:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829698.429984:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829698.429986:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829698.429988:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829698.429989:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829698.429991:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829698.429992:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829698.429994:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.429999:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.430001:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430006:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.430007:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829698.430008:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829698.430010:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829698.430012:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829698.430013:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298518546
00000100:00000001:0.0:1787829698.430014:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298519212
00000100:00000001:0.0:1787829698.430016:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298518546
00000100:00000001:0.0:1787829698.430017:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298519212
00000100:00000001:0.0:1787829698.430019:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829698.430020:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.430022:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829698.430024:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698]  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829698.430028:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829698.430029:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829698.430030:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829698.430031:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829698.430032:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829698.430033:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829698.430035:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829698.430040:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430044:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829698.430045:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430046:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.430047:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829698.430050:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430054:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829698.430057:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430061:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829698.430062:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430063:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.430064:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430065:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829698.430066:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829698.430067:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829698.430069:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829698.430070:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829698.430073:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829698.430077:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370368:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829698.430079:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829698.430081:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430085:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829698.430086:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.430087:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829698.430088:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829698.430090:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829698.430091:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370496:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829698.430093:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829698.430095:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829698.430096:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829698.430097:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829698.430099:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829698.430100:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829698.430102:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829698.430104:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829698.430105:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829698.430107:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370496, portal 4
00000100:00000001:0.0:1787829698.430108:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829698.430109:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:0.0:1787829698.430113:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430117:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829698.430119:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829698.430120:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829698.430121:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327370496, offset 0
00000400:00000200:0.0:1787829698.430123:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829698.430125:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829698.430127:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829698.430129:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829698.430131:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829698.430132:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829698.430134:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829698.430136:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40666:40666:256:4294967295] 192.168.203.116@tcp LPNI seq info [40666:40666:8:4294967295]
00000400:00000200:0.0:1787829698.430138:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829698.430142:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829698.430144:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829698.430146:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829698.430148:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829698.430150:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829698.430152:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829698.430167:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829698.430169:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829698.430172:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829698.430173:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829698.430176:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829698.430178:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829698.430179:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829698.430181:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829698.430183:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430189:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.430190:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430195:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.430196:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829698.430197:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829698.430199:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829698.430201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829698.430203:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298518551
00000100:00000001:0.0:1787829698.430204:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298519212
00000100:00000001:0.0:1787829698.430206:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298518551
00000100:00000001:0.0:1787829698.430207:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298519212
00000100:00000001:0.0:1787829698.430209:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829698.430210:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.430211:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829698.430213:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698]  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829698.430218:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829698.430218:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829698.430219:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829698.430221:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829698.430222:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829698.430223:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829698.430225:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829698.430230:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430234:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829698.430235:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430236:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.430237:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829698.430239:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430243:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829698.430245:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430249:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829698.430250:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430251:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829698.430252:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430253:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829698.430254:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829698.430256:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829698.430258:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829698.430259:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829698.430262:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829698.430267:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370496:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829698.430269:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430270:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.430272:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327370240/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430276:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829698.430278:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829698.430279:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829698.430280:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829698.430281:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829698.430282:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829698.430283:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829698.430285:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829698.430287:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829698.430288:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829698.430289:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:0.0:1787829698.430291:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829698.430292:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430293:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.430295:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327370368/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430299:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829698.430300:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829698.430301:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829698.430302:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829698.430303:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829698.430303:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829698.430304:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829698.430305:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829698.430306:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829698.430307:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829698.430308:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:0.0:1787829698.430309:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829698.430310:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430311:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829698.430313:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327370496/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829698.430317:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829698.430318:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829698.430319:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829698.430321:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829698.430322:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829698.430322:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829698.430324:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829698.430324:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829698.430325:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829698.430326:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829698.430327:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829698.430328:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829698.430329:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430330:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829698.430332:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829698.430333:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829698.430334:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829698.430336:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829698.940756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829698.940761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829698.940762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829698.940766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829698.940769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829698.940772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829698.940774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829698.940776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829698.940778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829699.132777:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298519212, at 4298519232
00000100:00000040:3.0:1787829699.132777:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298519212, at 4298519232
00000100:00000010:2.0:1787829699.132792:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00000100:00000010:3.0:1787829699.132798:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00010000:00000001:0.0:1787829699.260741:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829699.260748:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829699.260754:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829699.260755:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829699.260759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829699.260763:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829699.260765:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829699.260768:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829699.260771:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829699.260773:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829699.260775:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829699.260776:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829699.260829:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829699.260833:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829699.260837:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829699.964806:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829699.964813:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829699.964814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829699.964819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829699.964822:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829699.964825:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829699.964828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829699.964830:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829699.964833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829700.284790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829700.284795:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829700.284799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829700.284800:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829700.284803:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829700.284804:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829700.284884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829700.284888:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829700.284892:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829700.988769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829700.988776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829700.988778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829700.988783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829700.988786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829700.988789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829700.988792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829700.988794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829700.988796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829701.308783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829701.308788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829701.308794:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829701.308795:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829701.308798:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829701.308799:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829701.308855:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829701.308859:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829701.308862:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829702.012738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829702.012742:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829702.012744:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829702.012748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829702.012751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829702.012754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829702.012756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829702.012758:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829702.012760:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829702.332749:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829702.332756:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829702.332761:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829702.332762:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829702.332765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829702.332766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829702.332785:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829702.332787:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829702.332789:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829703.036772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829703.036777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829703.036779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829703.036783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829703.036786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829703.036789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829703.036791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829703.036793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829703.036795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829703.356752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829703.356758:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829703.356764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829703.356765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829703.356769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829703.356771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829703.356819:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829703.356822:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829703.356825:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829703.548760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829703.548768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829703.548771:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829703.548773:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829703.548777:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829703.548784:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3851
00000400:00000200:2.0:1787829703.548791:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829703.548796:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829703.548799:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829703.548801:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548803:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.548805:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548806:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829703.548809:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829703.548811:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829703.548813:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829703.548816:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829703.548817:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829703.548819:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829703.548821:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548822:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.548823:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548825:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829703.548827:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829703.548830:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548832:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829703.548834:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829703.548836:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829703.548838:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.548840:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829703.548841:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829703.548848:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829703.548850:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829703.548854:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829703.548855:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.548857:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829703.548859:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829703.548862:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829703.548868:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829703.548873:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829703.548880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829703.548888:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829703.548891:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829703.548895:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829703.548913:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829703.548915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829703.548918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829703.548919:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829703.548920:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829703.548922:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829703.548927:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3851
00000400:00000200:2.0:1787829703.548929:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829703.548931:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829703.548932:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829703.548934:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548936:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.548937:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548939:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829703.548941:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829703.548943:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829703.548944:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829703.548947:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829703.548949:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829703.548950:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829703.548953:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548955:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.548956:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548958:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829703.548961:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829703.548964:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829703.548965:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829703.548967:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829703.548971:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829703.548973:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787829703.548973:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.548975:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829703.548980:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829703.548981:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000008:00000001:2.0:1787829703.548982:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.548985:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.548986:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:2.0:1787829703.548987:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829703.548991:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829703.548993:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829703.548995:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.548996:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829703.548998:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829703.548998:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829703.549000:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829703.549003:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829703.549007:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829703.549015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829703.549018:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829703.549022:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.549024:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829703.549027:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.549029:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1787829703.549031:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829703.549034:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829703.549042:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.549042:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829703.549046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1787829703.549051:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327370624:192.168.203.116@tcp:250:kworker.0
00000100:00080000:2.0:1787829703.549051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829703.549053:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829703.549056:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829703.549060:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829703.549062:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787829703.549064:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3851
00000400:00000200:2.0:1787829703.549068:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:0.0:1787829703.549070:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000400:00000200:2.0:1787829703.549070:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
02000000:00000001:0.0:1787829703.549075:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829703.549075:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829703.549081:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829703.549083:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787829703.549085:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.549089:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000010:0.0:1787829703.549091:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
00000100:00000001:2.0:1787829703.549091:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829703.549095:0:36:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:0.0:1787829703.549097:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787829703.549098:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829703.549102:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829703.549105:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829703.549107:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787829703.549109:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:0.0:1787829703.549114:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000020:00000001:2.0:1787829703.549114:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829703.549121:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829703.549124:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:2.0:1787829703.549125:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.549130:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549132:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829703.549134:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370624, portal 25
00000020:00000040:2.0:1787829703.549134:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:0.0:1787829703.549140:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1787829703.549140:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829703.549145:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000100:00000001:2.0:1787829703.549146:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829703.549149:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829703.549154:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829703.549157:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829703.549159:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829703.549159:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829703.549162:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829703.549167:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.549171:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829703.549176:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000100:00000001:0.0:1787829703.549178:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1787829703.549179:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829703.549183:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829703.549185:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829703.549186:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549187:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829703.549190:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829703.549191:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829703.549193:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
02000000:00000010:2.0:1787829703.549195:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000200:0.0:1787829703.549198:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327370624, offset 0
00000100:00000001:2.0:1787829703.549201:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829703.549210:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829703.549211:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549222:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829703.549225:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829703.549226:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829703.549234:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829703.549235:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829703.549245:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787829703.549251:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829703.549252:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:2.0:1787829703.549255:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829703.549259:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1787829703.549260:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829703.549263:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829703.549266:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829703.549267:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549268:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829703.549273:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3851
00000400:00000200:0.0:1787829703.549277:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40667:40667:256:4294967295] 192.168.203.116@tcp LPNI seq info [40667:40667:8:4294967295]
00000400:00000200:2.0:1787829703.549279:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829703.549284:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829703.549286:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829703.549289:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829703.549290:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549294:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829703.549296:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549298:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829703.549301:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829703.549303:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829703.549306:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829703.549309:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000400:00000200:0.0:1787829703.549310:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1787829703.549311:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829703.549315:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829703.549322:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549325:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1787829703.549327:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787829703.549328:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549332:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829703.549334:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000010:0.0:1787829703.549336:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00080000:2.0:1787829703.549336:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549339:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829703.549342:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829703.549344:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829703.549347:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829703.549349:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829703.549349:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829703.549352:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829703.549355:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829703.549356:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787829703.549356:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829703.549360:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829703.549363:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829703.549367:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.1:1787829703.549367:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829703.549369:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829703.549372:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829703.549374:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.1:1787829703.549375:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
02000000:00000001:2.0:1787829703.549376:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829703.549381:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000800:00000100:0.0:1787829703.549383:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829703.549386:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829703.549387:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000100:00100000:2.0:1787829703.549392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829703.549396:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829703.549408:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829703.549409:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829703.549412:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829703.549419:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000100:00000040:2.0:1787829703.549419:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829703.549424:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:2.0:1787829703.549430:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.549432:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787829703.549434:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3861)
00000100:00000200:0.0:1787829703.549440:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549452:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.549457:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549465:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.549467:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829703.549473:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829703.549477:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829703.549480:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298524332
00000100:00000010:0.0:1787829703.549485:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc540.
00000100:00000001:0.0:1787829703.549492:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829703.549497:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298524332
00000100:00000010:0.0:1787829703.549517:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc8a0.
00000100:00000001:0.0:1787829703.549521:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829703.549525:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829703.549528:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.549531:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829703.549536:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703]  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829703.549547:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829703.549549:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829703.549551:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829703.549555:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829703.549557:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829703.549559:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829703.549563:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829703.549579:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549587:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829703.549589:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.549591:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.549594:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829703.549598:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549606:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829703.549611:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549619:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829703.549622:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.549624:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.549625:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.549628:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829703.549629:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829703.549633:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829703.549637:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829703.549640:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829703.549644:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829703.549653:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327370624:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829703.549691:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.549693:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.549697:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327370624/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549712:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829703.549714:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829703.549717:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829703.549720:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829703.549722:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829703.549724:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829703.549726:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829703.549729:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:0.0:1787829703.549733:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829703.549734:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829703.549737:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:0.0:1787829703.549740:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829703.549742:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.549744:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.549750:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829703.549753:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.549755:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829703.549757:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829703.549758:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829703.549762:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549770:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829703.549771:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.549773:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829703.549775:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829703.549777:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829703.549781:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370752:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829703.549785:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829703.549787:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829703.549788:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.549790:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829703.549803:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829703.549806:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829703.549810:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829703.549813:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829703.549817:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829703.549821:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370752, portal 10
00000100:00000001:0.0:1787829703.549822:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829703.549824:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:0.0:1787829703.549832:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549839:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829703.549841:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829703.549844:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829703.549847:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327370752, offset 0
00000400:00000200:0.0:1787829703.549851:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829703.549856:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829703.549860:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829703.549864:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829703.549867:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829703.549869:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829703.549872:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829703.549876:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40668:40668:256:4294967295] 192.168.203.116@tcp LPNI seq info [40668:40668:8:4294967295]
00000400:00000200:0.0:1787829703.549881:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829703.549888:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829703.549893:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829703.549897:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829703.549902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829703.549906:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829703.549911:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829703.549914:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829703.549919:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829703.549921:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829703.549924:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829703.549928:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829703.549932:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829703.549934:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829703.549937:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829703.549942:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549954:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.549958:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.549968:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.549970:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829703.549975:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829703.549979:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829703.549983:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829703.549986:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298523659
00000100:00000001:0.0:1787829703.549989:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298524332
00000100:00000001:0.0:1787829703.549992:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298523659
00000100:00000001:0.0:1787829703.549994:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298524332
00000100:00000001:0.0:1787829703.549997:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829703.550000:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.550002:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829703.550006:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703]  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829703.550016:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829703.550018:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829703.550020:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829703.550023:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829703.550025:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829703.550027:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829703.550031:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829703.550041:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550056:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829703.550058:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550061:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.550064:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829703.550069:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550077:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829703.550081:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550089:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829703.550092:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550094:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.550095:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550097:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829703.550099:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829703.550102:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829703.550105:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829703.550108:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829703.550113:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829703.550122:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370752:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829703.550126:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829703.550130:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550137:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829703.550138:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.550141:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829703.550143:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829703.550145:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829703.550148:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370880:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829703.550152:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829703.550154:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829703.550155:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.550157:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829703.550161:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829703.550163:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829703.550168:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829703.550171:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829703.550175:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829703.550179:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327370880, portal 4
00000100:00000001:0.0:1787829703.550181:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829703.550182:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:0.0:1787829703.550188:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550195:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829703.550198:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829703.550201:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829703.550204:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327370880, offset 0
00000400:00000200:0.0:1787829703.550208:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829703.550213:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829703.550216:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829703.550220:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829703.550223:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829703.550225:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829703.550228:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829703.550231:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40669:40669:256:4294967295] 192.168.203.116@tcp LPNI seq info [40669:40669:8:4294967295]
00000400:00000200:0.0:1787829703.550236:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829703.550243:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829703.550248:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829703.550252:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829703.550256:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829703.550260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829703.550264:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829703.550268:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829703.550272:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829703.550274:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829703.550278:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829703.550282:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829703.550285:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829703.550288:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829703.550291:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829703.550295:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550304:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.550307:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550314:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.550316:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829703.550319:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829703.550322:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829703.550326:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829703.550328:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298523666
00000100:00000001:0.0:1787829703.550331:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298524332
00000100:00000001:0.0:1787829703.550333:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298523666
00000100:00000001:0.0:1787829703.550335:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298524332
00000100:00000001:0.0:1787829703.550337:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829703.550340:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.550342:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829703.550346:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703]  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829703.550355:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829703.550357:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829703.550359:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829703.550361:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829703.550363:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829703.550366:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829703.550369:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829703.550378:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550385:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829703.550387:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550389:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.550390:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829703.550394:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550402:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829703.550407:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550415:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829703.550418:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550420:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.550421:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550423:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829703.550425:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829703.550428:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829703.550432:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829703.550434:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829703.550439:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829703.550448:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327370880:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829703.550453:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829703.550456:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550463:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829703.550465:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.550467:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829703.550469:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829703.550471:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829703.550474:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371008:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829703.550478:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829703.550479:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829703.550481:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829703.550483:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829703.550487:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829703.550489:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829703.550492:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829703.550495:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829703.550498:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829703.550518:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371008, portal 4
00000100:00000001:0.0:1787829703.550521:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829703.550522:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:0.0:1787829703.550529:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550537:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829703.550539:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829703.550542:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829703.550545:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327371008, offset 0
00000400:00000200:0.0:1787829703.550550:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829703.550554:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829703.550558:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829703.550561:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829703.550563:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829703.550566:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829703.550569:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829703.550572:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40670:40670:256:4294967295] 192.168.203.116@tcp LPNI seq info [40670:40670:8:4294967295]
00000400:00000200:0.0:1787829703.550577:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829703.550584:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829703.550590:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829703.550594:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829703.550598:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829703.550601:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829703.550605:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829703.550609:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829703.550613:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829703.550615:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829703.550618:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829703.550622:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829703.550625:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829703.550627:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829703.550630:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829703.550634:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550642:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.550646:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550653:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.550676:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829703.550682:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829703.550685:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829703.550689:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829703.550692:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298523672
00000100:00000001:0.0:1787829703.550694:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298524332
00000100:00000001:0.0:1787829703.550697:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298523672
00000100:00000001:0.0:1787829703.550699:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298524332
00000100:00000001:0.0:1787829703.550701:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829703.550703:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.550705:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829703.550710:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703]  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829703.550719:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829703.550721:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829703.550722:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829703.550725:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829703.550727:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829703.550729:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829703.550733:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829703.550744:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550751:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829703.550752:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550755:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.550756:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829703.550761:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550769:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829703.550773:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550781:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829703.550784:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550786:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829703.550787:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550789:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829703.550791:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829703.550794:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829703.550797:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829703.550800:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829703.550804:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829703.550813:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371008:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829703.550817:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550819:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.550823:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327370752/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550830:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829703.550832:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829703.550834:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829703.550836:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829703.550838:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829703.550840:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829703.550842:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829703.550844:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:0.0:1787829703.550847:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829703.550848:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829703.550850:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:0.0:1787829703.550853:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829703.550854:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550856:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.550860:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327370880/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550873:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829703.550874:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829703.550876:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829703.550879:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829703.550880:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829703.550882:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829703.550884:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829703.550886:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:0.0:1787829703.550888:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829703.550890:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829703.550892:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:0.0:1787829703.550894:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829703.550896:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550898:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829703.550902:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327371008/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829703.550910:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829703.550911:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829703.550913:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829703.550915:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829703.550917:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829703.550918:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829703.550920:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829703.550922:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:0.0:1787829703.550924:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829703.550925:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829703.550927:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:0.0:1787829703.550929:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829703.550931:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550933:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829703.550936:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829703.550938:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829703.550940:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829703.550942:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829704.060740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829704.060746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829704.060748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829704.060752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829704.060755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829704.060757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829704.060759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829704.060762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829704.060764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829704.252701:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298524332, at 4298524352
00000100:00000010:3.0:1787829704.252715:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000040:2.0:1787829704.252732:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298524332, at 4298524352
00000100:00000010:2.0:1787829704.252739:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00010000:00000001:0.0:1787829704.380743:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829704.380750:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829704.380752:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829704.380754:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380759:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829704.380761:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380762:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380764:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380766:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829704.380767:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829704.380768:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829704.380769:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380771:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829704.380771:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380772:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380774:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829704.380776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829704.380776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829704.380834:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829704.380838:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829704.380840:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829705.084787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829705.084795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829705.084797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829705.084802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829705.084806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829705.084809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829705.084812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829705.084814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829705.084817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829705.404729:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829705.404734:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829705.404738:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829705.404739:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829705.404741:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829705.404742:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829705.404759:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829705.404761:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829705.404763:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829706.108738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829706.108744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829706.108746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829706.108750:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829706.108753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829706.108757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829706.108759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829706.108760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829706.108762:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829706.428790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829706.428798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829706.428804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829706.428806:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829706.428808:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829706.428810:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829706.428921:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829706.428926:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829706.428930:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829707.132762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829707.132768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829707.132769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829707.132773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829707.132776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829707.132779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829707.132781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829707.132783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829707.132785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829707.452759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829707.452769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829707.452775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829707.452777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829707.452781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829707.452782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829707.452848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829707.452853:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829707.452858:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829708.156763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829708.156770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829708.156773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829708.156778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829708.156781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829708.156785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829708.156787:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829708.156790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829708.156792:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829708.476771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829708.476778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829708.476786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829708.476788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829708.476791:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829708.476792:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829708.476814:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829708.476818:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829708.476821:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829708.668841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829708.668850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829708.668852:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829708.668855:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829708.668859:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829708.668864:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3856
00000400:00000200:3.0:1787829708.668871:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829708.668876:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829708.668879:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829708.668881:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.668883:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829708.668885:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.668887:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829708.668890:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829708.668891:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829708.668893:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829708.668896:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829708.668897:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829708.668899:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829708.668901:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.668902:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829708.668903:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.668905:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829708.668907:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829708.668909:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829708.668911:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829708.668913:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829708.668916:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829708.668918:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.668920:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829708.668922:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829708.668930:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000020:00000040:3.0:1787829708.668933:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829708.668937:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829708.668938:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.668940:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829708.668942:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829708.668946:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829708.668952:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829708.668956:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829708.668963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829708.668970:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829708.668974:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829708.668977:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829708.668998:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829708.669000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829708.669002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829708.669003:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829708.669005:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829708.669007:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829708.669008:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3856
00000400:00000200:3.0:1787829708.669010:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829708.669012:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829708.669013:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829708.669015:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669017:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829708.669019:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669021:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829708.669022:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829708.669024:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829708.669025:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829708.669026:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829708.669028:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829708.669029:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829708.669031:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669033:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829708.669034:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669036:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829708.669037:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829708.669039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669041:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829708.669042:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829708.669046:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829708.669047:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669049:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829708.669050:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00100000:3.0:1787829708.669052:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829708.669057:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000008:00000001:3.0:1787829708.669057:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669061:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.669063:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:3.0:1787829708.669065:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829708.669067:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829708.669069:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829708.669071:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669073:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829708.669074:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829708.669074:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829708.669075:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829708.669080:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829708.669084:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829708.669091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829708.669092:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829708.669096:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.669102:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829708.669106:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829708.669107:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829708.669112:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669112:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829708.669119:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829708.669122:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327371136:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829708.669131:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829708.669131:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829708.669134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829708.669137:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1787829708.669138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829708.669141:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:0.0:1787829708.669142:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829708.669143:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:0.0:1787829708.669146:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829708.669147:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829708.669151:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3856
00000400:00000200:3.0:1787829708.669153:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:0.0:1787829708.669154:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
00000400:00000200:3.0:1787829708.669156:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
02000000:00000001:0.0:1787829708.669159:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829708.669160:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829708.669165:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669167:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829708.669171:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829708.669171:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669176:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:0.0:1787829708.669178:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000020:00000001:3.0:1787829708.669179:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829708.669182:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829708.669184:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1787829708.669187:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000020:00000040:3.0:1787829708.669187:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829708.669193:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829708.669196:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:0.0:1787829708.669197:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371136, portal 25
00000100:00000001:0.0:1787829708.669203:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1787829708.669203:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829708.669206:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000001:3.0:1787829708.669208:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829708.669214:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829708.669217:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829708.669219:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829708.669226:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829708.669233:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669239:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829708.669244:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829708.669248:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829708.669249:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787829708.669253:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829708.669257:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669257:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829708.669259:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1787829708.669263:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000008:00000001:3.0:1787829708.669263:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669268:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829708.669270:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327371136, offset 0
00000100:00000010:3.0:1787829708.669274:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829708.669277:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829708.669282:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829708.669283:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829708.669285:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669288:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829708.669290:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829708.669291:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.669295:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:3.0:1787829708.669296:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829708.669301:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.669306:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00100000:3.0:1787829708.669307:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829708.669314:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787829708.669317:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829708.669320:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.669321:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000040:3.0:1787829708.669326:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829708.669329:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829708.669335:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669341:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.669344:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40671:40671:256:4294967295] 192.168.203.116@tcp LPNI seq info [40671:40671:8:4294967295]
00000100:00080000:3.0:1787829708.669345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829708.669354:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829708.669356:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829708.669357:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829708.669358:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829708.669361:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829708.669364:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3856
00000400:00000200:0.0:1787829708.669371:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787829708.669379:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:0.0:1787829708.669381:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787829708.669385:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829708.669388:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000010:0.0:1787829708.669390:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00000040:3.0:1787829708.669390:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669396:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829708.669397:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669399:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829708.669402:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.0:1787829708.669404:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787829708.669404:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829708.669407:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829708.669410:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:0.0:1787829708.669411:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000020:00000001:3.0:1787829708.669412:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829708.669416:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829708.669421:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669425:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787829708.669426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829708.669428:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669432:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829708.669434:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000020:00000040:3.0:1787829708.669435:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829708.669439:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829708.669442:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829708.669446:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829708.669446:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829708.669449:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:0.0:1787829708.669451:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
02000000:00000001:3.0:1787829708.669453:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669456:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829708.669457:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829708.669459:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:3.0:1787829708.669459:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669464:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829708.669483:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829708.669485:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829708.669487:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829708.669488:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829708.669489:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829708.669491:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829708.669492:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829708.669495:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829708.669497:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829708.669687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829708.669693:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829708.669695:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.669699:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1787829708.669700:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829708.669707:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000100:00000001:3.0:1787829708.669709:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829708.669712:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000040:3.0:1787829708.669714:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3866)
00000100:00000001:0.0:1787829708.669719:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829708.669726:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.669738:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.669742:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.669757:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.669759:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829708.669765:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.669769:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829708.669772:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298529453
00000100:00000010:0.0:1787829708.669778:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc1e0.
00000100:00000001:0.0:1787829708.669784:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829708.669789:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298529453
00000100:00000010:0.0:1787829708.669792:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccb40.
00000100:00000001:0.0:1787829708.669796:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829708.669805:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829708.669808:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.669811:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829708.669816:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708]  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829708.669827:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829708.669829:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829708.669831:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829708.669834:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829708.669836:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829708.669839:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829708.669843:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829708.669853:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.669861:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829708.669863:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.669865:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.669867:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829708.669871:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.669880:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829708.669885:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.669893:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829708.669896:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.669898:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.669899:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.669901:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829708.669903:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829708.669906:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829708.669910:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829708.669912:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829708.669917:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829708.669926:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327371136:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829708.669932:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.669934:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.669939:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327371136/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.669946:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829708.669948:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829708.669951:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829708.669954:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829708.669956:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829708.669957:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829708.669959:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829708.669962:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:0.0:1787829708.669966:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829708.669968:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829708.669970:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:0.0:1787829708.669975:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829708.669976:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.669979:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.669985:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829708.669987:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.669989:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829708.669991:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829708.669992:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829708.669996:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670004:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829708.670006:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.670008:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829708.670010:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829708.670012:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829708.670015:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371264:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829708.670019:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829708.670021:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829708.670023:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.670025:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829708.670028:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829708.670030:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.670034:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829708.670037:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829708.670040:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829708.670043:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371264, portal 10
00000100:00000001:0.0:1787829708.670045:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829708.670047:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:0.0:1787829708.670054:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670061:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829708.670064:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829708.670066:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829708.670069:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327371264, offset 0
00000400:00000200:0.0:1787829708.670073:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829708.670078:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829708.670082:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829708.670085:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829708.670088:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829708.670091:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829708.670094:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829708.670097:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40672:40672:256:4294967295] 192.168.203.116@tcp LPNI seq info [40672:40672:8:4294967295]
00000400:00000200:0.0:1787829708.670101:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829708.670108:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829708.670114:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829708.670118:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829708.670123:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829708.670126:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829708.670131:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829708.670135:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829708.670139:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829708.670141:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829708.670144:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829708.670148:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829708.670151:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829708.670153:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829708.670155:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829708.670159:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670168:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.670171:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670179:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.670181:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829708.670184:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829708.670188:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829708.670191:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829708.670194:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298528779
00000100:00000001:0.0:1787829708.670197:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298529453
00000100:00000001:0.0:1787829708.670199:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298528779
00000100:00000001:0.0:1787829708.670201:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298529453
00000100:00000001:0.0:1787829708.670203:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829708.670206:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.670208:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829708.670212:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708]  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829708.670220:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829708.670222:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829708.670224:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829708.670226:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829708.670228:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829708.670230:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829708.670234:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829708.670243:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670250:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829708.670252:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670254:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.670255:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829708.670259:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670266:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829708.670270:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670279:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829708.670282:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670283:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.670285:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670287:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829708.670289:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829708.670292:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829708.670296:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829708.670299:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829708.670303:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829708.670312:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371264:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829708.670317:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829708.670320:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670328:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829708.670330:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.670332:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829708.670333:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829708.670336:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829708.670339:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371392:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829708.670343:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829708.670344:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829708.670346:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.670348:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829708.670352:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829708.670354:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.670358:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829708.670361:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829708.670364:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829708.670367:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371392, portal 4
00000100:00000001:0.0:1787829708.670369:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829708.670371:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:0.0:1787829708.670376:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670383:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829708.670386:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829708.670388:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829708.670391:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327371392, offset 0
00000400:00000200:0.0:1787829708.670395:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829708.670399:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829708.670403:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829708.670405:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829708.670408:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829708.670411:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829708.670414:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829708.670417:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40673:40673:256:4294967295] 192.168.203.116@tcp LPNI seq info [40673:40673:8:4294967295]
00000400:00000200:0.0:1787829708.670422:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829708.670429:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829708.670434:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829708.670437:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829708.670441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829708.670444:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829708.670449:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829708.670452:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829708.670456:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829708.670458:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829708.670461:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829708.670476:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829708.670480:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829708.670483:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829708.670486:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829708.670491:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670500:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.670503:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670510:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.670512:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829708.670515:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829708.670518:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829708.670521:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829708.670523:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298528786
00000100:00000001:0.0:1787829708.670526:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298529453
00000100:00000001:0.0:1787829708.670528:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298528786
00000100:00000001:0.0:1787829708.670530:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298529453
00000100:00000001:0.0:1787829708.670533:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829708.670535:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.670537:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829708.670541:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708]  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829708.670549:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829708.670551:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829708.670553:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829708.670555:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829708.670557:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829708.670559:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829708.670563:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829708.670572:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670585:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829708.670587:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670589:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.670590:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829708.670594:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670602:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829708.670606:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670613:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829708.670615:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670617:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.670618:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670620:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829708.670622:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829708.670625:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829708.670628:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829708.670630:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829708.670635:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829708.670643:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371392:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829708.670647:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829708.670651:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670678:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829708.670680:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.670683:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829708.670685:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829708.670687:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829708.670690:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371520:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829708.670694:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829708.670696:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829708.670697:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829708.670699:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829708.670703:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829708.670705:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829708.670709:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829708.670711:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829708.670714:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829708.670717:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371520, portal 4
00000100:00000001:0.0:1787829708.670720:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829708.670721:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829708.670727:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670734:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829708.670737:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829708.670739:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829708.670742:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327371520, offset 0
00000400:00000200:0.0:1787829708.670746:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829708.670750:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829708.670753:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829708.670756:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829708.670759:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829708.670762:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829708.670764:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829708.670768:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40674:40674:256:4294967295] 192.168.203.116@tcp LPNI seq info [40674:40674:8:4294967295]
00000400:00000200:0.0:1787829708.670772:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829708.670779:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829708.670785:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829708.670788:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829708.670793:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829708.670795:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829708.670800:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829708.670803:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829708.670807:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829708.670809:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829708.670812:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829708.670816:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829708.670819:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829708.670821:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829708.670824:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829708.670828:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670836:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.670840:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670847:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.670849:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829708.670852:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829708.670855:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829708.670858:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829708.670861:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298528792
00000100:00000001:0.0:1787829708.670863:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298529453
00000100:00000001:0.0:1787829708.670866:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298528792
00000100:00000001:0.0:1787829708.670868:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298529453
00000100:00000001:0.0:1787829708.670870:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829708.670872:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.670874:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829708.670878:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708]  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829708.670888:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829708.670889:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829708.670891:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829708.670893:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829708.670895:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829708.670898:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829708.670901:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829708.670910:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670917:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829708.670919:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670920:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.670922:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829708.670925:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670932:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829708.670936:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670944:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829708.670947:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670949:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829708.670950:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670952:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829708.670953:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829708.670956:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829708.670959:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829708.670961:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829708.670966:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829708.670975:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371520:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829708.670979:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.670981:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.670984:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327371264/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.670991:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829708.670992:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829708.670994:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829708.670997:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829708.670998:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829708.671000:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829708.671002:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829708.671004:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:0.0:1787829708.671007:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829708.671009:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829708.671010:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:0.0:1787829708.671013:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829708.671014:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.671017:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.671021:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327371392/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.671029:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829708.671030:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829708.671032:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829708.671034:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829708.671036:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829708.671037:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829708.671039:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829708.671041:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:0.0:1787829708.671043:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829708.671045:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829708.671047:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829708.671049:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829708.671051:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.671052:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829708.671056:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327371520/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829708.671063:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829708.671065:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829708.671066:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829708.671068:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829708.671070:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829708.671071:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829708.671073:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829708.671074:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:0.0:1787829708.671076:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829708.671077:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829708.671078:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829708.671080:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829708.671082:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.671084:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829708.671087:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829708.671088:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829708.671090:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829708.671092:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829709.180813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829709.180821:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829709.180823:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829709.180828:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829709.180831:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829709.180835:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829709.180838:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829709.180840:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829709.180843:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829709.372714:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298529453, at 4298529472
00000100:00000010:3.0:1787829709.372727:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000040:2.0:1787829709.372766:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298529453, at 4298529472
00000100:00000010:2.0:1787829709.372779:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00010000:00000001:0.0:1787829709.500768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829709.500775:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829709.500782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829709.500784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829709.500786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829709.500788:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829709.500790:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829709.500793:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829709.500795:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829709.500796:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829709.500798:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829709.500799:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829709.500859:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829709.500865:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829709.500870:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829710.204738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829710.204746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829710.204747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829710.204753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829710.204756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829710.204759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829710.204762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829710.204764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829710.204766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829710.524785:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829710.524794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829710.524801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829710.524803:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829710.524807:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829710.524808:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829710.524846:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829710.524851:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829710.524855:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829711.228831:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829711.228839:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829711.228847:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829711.228852:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829711.228856:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829711.228859:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829711.228862:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829711.228865:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829711.228868:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829711.548779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829711.548786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829711.548793:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829711.548794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829711.548798:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829711.548800:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829711.548823:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829711.548826:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829711.548830:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829712.252744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829712.252752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829712.252754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829712.252759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829712.252763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829712.252766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829712.252769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829712.252777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829712.252781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829712.572757:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829712.572762:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829712.572768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829712.572769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829712.572772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829712.572773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829712.572875:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829712.572880:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829712.572891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829713.276758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829713.276764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829713.276766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829713.276770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829713.276772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829713.276775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.276777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829713.276779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.276781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829713.596791:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829713.596798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829713.596803:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829713.596805:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829713.596808:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829713.596809:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829713.596841:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829713.596846:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829713.596851:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829713.788794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829713.788805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829713.788808:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829713.788812:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829713.788817:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829713.788824:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3861
00000400:00000200:1.0:1787829713.788834:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829713.788841:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829713.788844:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829713.788846:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.788849:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829713.788851:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.788853:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829713.788857:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829713.788859:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829713.788862:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829713.788865:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829713.788867:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829713.788868:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829713.788870:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.788872:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829713.788873:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.788874:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829713.788883:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829713.788886:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829713.788888:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829713.788891:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829713.788894:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829713.788897:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829713.788899:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829713.788901:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829713.788911:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829713.788914:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829713.788918:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829713.788920:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829713.788922:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829713.788925:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829713.788930:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829713.788937:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829713.788944:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829713.788953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829713.788962:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829713.788967:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829713.788971:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829713.788992:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829713.788994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829713.788997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829713.788998:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829713.788999:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829713.789001:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829713.789003:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3861
00000400:00000200:1.0:1787829713.789005:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829713.789007:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829713.789008:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829713.789010:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789012:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829713.789014:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789015:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829713.789017:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829713.789018:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829713.789019:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829713.789021:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829713.789022:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829713.789024:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829713.789026:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789027:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829713.789029:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789030:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829713.789033:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829713.789035:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789039:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829713.789040:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829713.789042:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:1.0:1787829713.789046:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829713.789049:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789050:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829713.789051:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829713.789056:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:1.0:1787829713.789058:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829713.789061:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829713.789064:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829713.789066:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787829713.789067:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829713.789072:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829713.789078:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829713.789080:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789083:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829713.789084:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829713.789087:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829713.789088:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829713.789091:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.789093:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:1.0:1787829713.789095:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829713.789097:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829713.789100:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.789101:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829713.789106:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327371648:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829713.789110:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:1.0:1787829713.789114:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829713.789115:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829713.789119:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.789124:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829713.789128:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829713.789130:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829713.789131:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829713.789134:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829713.789135:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829713.789143:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829713.789149:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000100:00000001:1.0:1787829713.789151:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829713.789154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787829713.789157:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00080000:1.0:1787829713.789158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829713.789161:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829713.789163:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829713.789166:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:0.0:1787829713.789167:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371648, portal 25
00000100:00000001:0.0:1787829713.789171:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829713.789173:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00080000:1.0:1787829713.789174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3861
00000400:00000200:1.0:1787829713.789180:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829713.789184:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829713.789187:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829713.789191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829713.789198:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789203:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829713.789204:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829713.789206:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829713.789212:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789213:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:0.0:1787829713.789217:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000020:00000001:1.0:1787829713.789217:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829713.789219:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829713.789222:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:0.0:1787829713.789223:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327371648, offset 0
00000020:00000040:1.0:1787829713.789224:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829713.789227:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829713.789229:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829713.789234:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829713.789236:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789239:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829713.789241:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789242:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829713.789244:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829713.789245:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829713.789248:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789251:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829713.789253:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829713.789255:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829713.789256:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000001:1.0:1787829713.789260:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829713.789263:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829713.789265:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:1.0:1787829713.789266:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829713.789270:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.789272:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:1.0:1787829713.789273:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.789277:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000010:1.0:1787829713.789278:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000400:00000200:0.0:1787829713.789281:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000040:1.0:1787829713.789283:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829713.789289:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40675:40675:256:4294967295] 192.168.203.116@tcp LPNI seq info [40675:40675:8:4294967295]
02000000:00000001:1.0:1787829713.789289:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829713.789292:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.789296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829713.789296:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829713.789300:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829713.789302:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829713.789305:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:1.0:1787829713.789310:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.789312:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00100000:1.0:1787829713.789317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829713.789323:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789331:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000010:0.0:1787829713.789332:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00000001:1.0:1787829713.789333:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829713.789339:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829713.789345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829713.789351:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1787829713.789352:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829713.789357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829713.789362:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829713.789363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829713.789368:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1787829713.789369:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00080000:1.0:1787829713.789372:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000100:0.0:1787829713.789377:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789380:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:0.0:1787829713.789381:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829713.789388:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1787829713.789388:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3861
00000400:00000200:0.0:1787829713.789398:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829713.789399:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.789404:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829713.789407:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000400:00000200:1.0:1787829713.789408:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829713.789411:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:1.0:1787829713.789413:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1787829713.789418:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829713.789422:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789442:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829713.789447:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789452:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829713.789457:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:1.0:1787829713.789457:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829713.789459:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000040:0.0:1787829713.789464:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829713.789464:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829713.789468:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829713.789475:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829713.789476:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789479:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:1.0:1787829713.789482:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829713.789487:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829713.789490:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829713.789495:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829713.789497:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829713.789500:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298534572
00000100:00000040:1.0:1787829713.789504:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:0.0:1787829713.789507:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc480.
00000100:00000001:1.0:1787829713.789509:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829713.789514:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829713.789516:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:1.0:1787829713.789519:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829713.789525:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829713.789527:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829713.789531:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298534572
02000000:00000001:1.0:1787829713.789531:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1787829713.789537:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc060.
02000000:00000001:1.0:1787829713.789538:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789541:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829713.789543:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829713.789547:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829713.789548:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1787829713.789554:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789555:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789560:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829713.789560:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829713.789567:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713]  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787829713.789568:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829713.789575:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000001:0.0:1787829713.789582:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:1.0:1787829713.789582:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:0.0:1787829713.789586:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787829713.789586:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829713.789589:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000100:00000001:1.0:1787829713.789591:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1787829713.789594:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
02000000:00000001:1.0:1787829713.789595:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1787829713.789598:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
02000000:00000001:1.0:1787829713.789600:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789601:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829713.789606:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787829713.789609:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000200:0.0:1787829713.789616:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829713.789618:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789626:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829713.789628:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.789631:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:1.0:1787829713.789632:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829713.789633:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829713.789640:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829713.789682:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829713.789686:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829713.789694:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.789695:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829713.789708:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829713.789709:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789718:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1787829713.789720:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3871)
00000100:00000001:0.0:1787829713.789722:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.789725:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.789726:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.789728:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829713.789730:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829713.789733:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829713.789736:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829713.789744:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829713.789749:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829713.789758:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327371648:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829713.789764:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.789766:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.789770:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327371648/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.789778:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829713.789782:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829713.789785:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829713.789788:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829713.789790:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829713.789793:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829713.789795:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829713.789798:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829713.789802:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829713.789804:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829713.789806:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829713.789808:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829713.789809:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.789811:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.789818:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829713.789819:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789821:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829713.789823:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829713.789824:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829713.789827:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.789832:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829713.789833:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.789835:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829713.789837:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829713.789838:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829713.789840:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371776:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829713.789842:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829713.789844:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829713.789845:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.789847:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829713.789849:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829713.789850:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.789853:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829713.789855:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829713.789858:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829713.789861:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371776, portal 10
00000100:00000001:0.0:1787829713.789862:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829713.789863:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829713.789870:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.789875:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829713.789877:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829713.789879:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829713.789880:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327371776, offset 0
00000400:00000200:0.0:1787829713.789883:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829713.789886:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.789889:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829713.789891:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829713.789893:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829713.789895:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829713.789896:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829713.789899:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40676:40676:256:4294967295] 192.168.203.116@tcp LPNI seq info [40676:40676:8:4294967295]
00000400:00000200:0.0:1787829713.789902:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.789907:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829713.789910:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829713.789912:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829713.789915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829713.789917:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829713.789920:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829713.789922:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1787829713.789925:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829713.789926:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829713.789929:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829713.789931:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829713.789933:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829713.789935:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829713.789936:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829713.789939:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.789945:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.789947:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.789957:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789958:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829713.789960:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829713.789963:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829713.789965:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829713.789967:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298533899
00000100:00000001:0.0:1787829713.789969:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298534572
00000100:00000001:0.0:1787829713.789971:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298533899
00000100:00000001:0.0:1787829713.789972:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298534572
00000100:00000001:0.0:1787829713.789974:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829713.789975:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.789976:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829713.789978:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713]  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829713.789983:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829713.789984:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829713.789986:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829713.789987:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829713.789988:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829713.789990:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829713.789992:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829713.789997:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790002:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829713.790003:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790004:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.790005:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829713.790007:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790016:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829713.790020:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790028:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829713.790030:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790032:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.790034:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790036:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829713.790039:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829713.790042:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829713.790046:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829713.790049:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829713.790053:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829713.790063:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371776:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829713.790068:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829713.790071:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790078:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829713.790079:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.790081:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829713.790083:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829713.790084:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829713.790086:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371904:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829713.790089:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829713.790090:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829713.790092:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.790093:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829713.790096:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829713.790097:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.790100:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829713.790102:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829713.790105:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829713.790107:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327371904, portal 4
00000100:00000001:0.0:1787829713.790108:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829713.790110:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829713.790114:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790119:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829713.790120:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829713.790122:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829713.790123:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327371904, offset 0
00000400:00000200:0.0:1787829713.790126:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829713.790129:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.790131:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829713.790133:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829713.790135:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829713.790137:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829713.790138:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829713.790141:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40677:40677:256:4294967295] 192.168.203.116@tcp LPNI seq info [40677:40677:8:4294967295]
00000400:00000200:0.0:1787829713.790144:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.790148:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829713.790151:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829713.790153:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829713.790156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829713.790158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829713.790161:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829713.790163:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1787829713.790166:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829713.790168:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829713.790170:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829713.790173:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829713.790175:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829713.790176:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829713.790177:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829713.790180:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790185:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.790187:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790192:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.790193:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829713.790195:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829713.790197:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829713.790199:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829713.790201:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298533906
00000100:00000001:0.0:1787829713.790203:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298534572
00000100:00000001:0.0:1787829713.790204:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298533906
00000100:00000001:0.0:1787829713.790206:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298534572
00000100:00000001:0.0:1787829713.790207:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829713.790208:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.790209:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829713.790211:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713]  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829713.790217:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829713.790217:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829713.790219:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829713.790220:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829713.790221:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829713.790223:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829713.790225:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829713.790231:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790235:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829713.790236:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790237:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.790237:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829713.790240:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790243:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829713.790246:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790250:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829713.790251:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790252:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.790253:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790254:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829713.790255:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829713.790257:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829713.790259:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829713.790260:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829713.790263:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829713.790268:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327371904:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829713.790270:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829713.790272:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790278:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829713.790279:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.790280:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829713.790282:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829713.790283:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829713.790285:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372032:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829713.790286:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829713.790287:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829713.790288:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829713.790289:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829713.790291:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829713.790293:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829713.790294:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829713.790296:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829713.790297:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829713.790299:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372032, portal 4
00000100:00000001:0.0:1787829713.790300:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829713.790302:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:0.0:1787829713.790306:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790310:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829713.790311:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829713.790313:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829713.790314:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327372032, offset 0
00000400:00000200:0.0:1787829713.790317:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829713.790318:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.790320:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829713.790322:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829713.790324:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829713.790325:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829713.790326:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829713.790329:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40678:40678:256:4294967295] 192.168.203.116@tcp LPNI seq info [40678:40678:8:4294967295]
00000400:00000200:0.0:1787829713.790332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829713.790336:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829713.790339:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829713.790341:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829713.790343:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829713.790345:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829713.790347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829713.790364:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1787829713.790367:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829713.790369:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829713.790371:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829713.790374:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829713.790376:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829713.790377:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829713.790379:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829713.790381:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790387:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.790389:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790393:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.790394:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829713.790396:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829713.790398:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829713.790400:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829713.790401:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298533911
00000100:00000001:0.0:1787829713.790402:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298534572
00000100:00000001:0.0:1787829713.790404:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298533911
00000100:00000001:0.0:1787829713.790405:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298534572
00000100:00000001:0.0:1787829713.790407:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829713.790409:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.790410:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829713.790412:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713]  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829713.790418:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829713.790419:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829713.790420:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829713.790421:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829713.790422:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829713.790423:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829713.790425:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829713.790446:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790451:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829713.790453:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790455:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.790456:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829713.790458:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790463:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829713.790465:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790469:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829713.790470:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790471:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829713.790472:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790473:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829713.790475:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829713.790476:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829713.790479:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829713.790480:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829713.790483:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829713.790488:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372032:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829713.790490:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790491:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.790493:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327371776/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790497:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829713.790498:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829713.790500:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829713.790501:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829713.790503:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829713.790504:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829713.790505:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829713.790507:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829713.790508:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829713.790509:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829713.790510:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829713.790512:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829713.790513:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790514:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.790516:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327371904/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790521:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829713.790522:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829713.790522:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829713.790524:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829713.790525:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829713.790525:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829713.790526:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829713.790528:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829713.790530:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829713.790530:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829713.790531:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829713.790534:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829713.790535:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790535:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829713.790537:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327372032/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829713.790541:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829713.790542:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829713.790542:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829713.790544:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829713.790544:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829713.790545:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829713.790546:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829713.790547:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829713.790548:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829713.790549:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829713.790550:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829713.790551:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829713.790551:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790553:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829713.790555:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829713.790556:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829713.790557:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829713.790558:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829714.300776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829714.300781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829714.300783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829714.300787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829714.300790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829714.300793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829714.300795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829714.300797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829714.300799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829714.492782:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298534572, at 4298534592
00000100:00000040:3.0:1787829714.492782:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298534572, at 4298534592
00000100:00000010:3.0:1787829714.492800:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:2.0:1787829714.492804:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00010000:00000001:0.0:1787829714.620798:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829714.620811:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829714.620814:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829714.620819:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620827:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829714.620829:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620832:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620835:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620840:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829714.620842:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829714.620843:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829714.620845:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620847:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829714.620849:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620858:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620861:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829714.620866:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829714.620867:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829714.620887:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829714.620889:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829714.620891:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829715.324817:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829715.324827:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829715.324831:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829715.324838:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829715.324842:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829715.324848:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829715.324851:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829715.324856:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829715.324860:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829715.644783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829715.644790:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829715.644796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829715.644797:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829715.644801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829715.644802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829715.644868:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829715.644872:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829715.644875:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829716.348748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829716.348753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829716.348754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829716.348759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829716.348762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829716.348765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829716.348766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829716.348768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829716.348770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829716.668751:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829716.668756:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829716.668761:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829716.668762:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829716.668765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829716.668766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829716.668808:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829716.668812:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829716.668816:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829717.372755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829717.372762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829717.372764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829717.372769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829717.372784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829717.372791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829717.372794:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829717.372797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829717.372801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829717.692759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829717.692764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829717.692768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829717.692770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829717.692773:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829717.692774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829717.692791:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829717.692793:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829717.692795:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829718.396765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829718.396770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829718.396772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829718.396776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829718.396779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829718.396782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.396783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829718.396786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.396788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829718.716757:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829718.716764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829718.716770:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829718.716771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829718.716775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829718.716776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829718.716821:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829718.716824:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829718.716827:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829718.908850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829718.908862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829718.908865:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829718.908869:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829718.908874:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829718.908883:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3866
00000400:00000200:1.0:1787829718.908892:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829718.908916:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829718.908920:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829718.908925:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.908929:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829718.908933:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.908936:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829718.908941:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829718.908943:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829718.908953:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829718.908956:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829718.908958:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829718.908961:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829718.908964:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.908965:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829718.908968:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.908969:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829718.908972:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829718.908975:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829718.908977:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829718.908979:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829718.908982:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829718.908985:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829718.908986:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829718.908988:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829718.908996:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829718.908999:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829718.909002:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829718.909003:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829718.909005:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829718.909007:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829718.909010:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829718.909024:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829718.909030:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829718.909040:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829718.909050:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829718.909053:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829718.909057:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829718.909074:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829718.909077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829718.909081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829718.909082:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829718.909084:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829718.909087:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829718.909090:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3866
00000400:00000200:1.0:1787829718.909095:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829718.909099:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829718.909102:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829718.909106:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909109:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829718.909111:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829718.909126:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829718.909126:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829718.909131:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787829718.909133:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829718.909134:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787829718.909136:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829718.909142:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1787829718.909143:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829718.909146:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829718.909152:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829718.909157:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1787829718.909158:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829718.909160:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.909166:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:1.0:1787829718.909168:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829718.909170:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829718.909174:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829718.909177:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327372160:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829718.909180:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829718.909181:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829718.909187:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:1.0:1787829718.909187:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829718.909189:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.909194:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829718.909194:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829718.909199:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000010:0.0:1787829718.909200:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829718.909204:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829718.909206:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829718.909211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829718.909212:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829718.909215:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
02000000:00000001:1.0:1787829718.909217:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1787829718.909221:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
02000000:00000001:1.0:1787829718.909225:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1787829718.909229:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372160, portal 25
02000000:00000001:1.0:1787829718.909230:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909233:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829718.909236:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829718.909242:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000008:00100000:1.0:1787829718.909242:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829718.909249:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829718.909250:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829718.909255:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909262:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829718.909265:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000010:1.0:1787829718.909265:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000400:00000010:0.0:1787829718.909271:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000020:00000040:1.0:1787829718.909272:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000200:0.0:1787829718.909276:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327372160, offset 0
02000000:00000001:1.0:1787829718.909278:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829718.909281:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.909284:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909286:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829718.909289:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829718.909292:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.909293:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:1.0:1787829718.909300:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000400:00000200:0.0:1787829718.909301:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829718.909308:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829718.909309:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.909313:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829718.909320:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00100000:1.0:1787829718.909322:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829718.909323:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829718.909331:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40679:40679:256:4294967295] 192.168.203.116@tcp LPNI seq info [40679:40679:8:4294967295]
00000400:00000200:0.0:1787829718.909341:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829718.909348:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829718.909354:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.909355:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787829718.909363:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829718.909364:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.909375:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00000001:1.0:1787829718.909377:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829718.909382:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1787829718.909383:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829718.909390:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000100:00080000:1.0:1787829718.909391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829718.909412:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1787829718.909412:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829718.909417:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1787829718.909418:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829718.909425:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909425:0:37:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:0.0:1787829718.909430:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000100:00080000:1.0:1787829718.909433:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3866
00000400:00000200:0.0:1787829718.909435:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829718.909442:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.909443:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829718.909451:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000200:1.0:1787829718.909453:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000010:0.0:1787829718.909454:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00080000:1.0:1787829718.909460:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829718.909462:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787829718.909470:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829718.909471:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829718.909477:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829718.909483:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909488:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829718.909489:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:1.0:1787829718.909491:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829718.909495:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000040:0.0:1787829718.909497:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829718.909498:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829718.909502:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829718.909508:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829718.909513:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:0.0:1787829718.909516:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909519:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787829718.909521:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909525:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829718.909527:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829718.909529:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909534:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829718.909536:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:1.0:1787829718.909541:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:0.0:1787829718.909543:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298539692
00000100:00080000:1.0:1787829718.909549:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000010:0.0:1787829718.909551:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf00.
00000100:00000001:1.0:1787829718.909554:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829718.909558:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829718.909561:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:1.0:1787829718.909563:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829718.909568:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829718.909570:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829718.909573:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829718.909577:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298539692
00000008:00000001:1.0:1787829718.909577:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829718.909582:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829718.909585:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc720.
00000100:00000010:1.0:1787829718.909589:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829718.909591:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000040:1.0:1787829718.909595:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:0.0:1787829718.909598:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829718.909601:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829718.909604:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829718.909605:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909609:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829718.909610:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829718.909614:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829718.909617:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829718.909619:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829718]  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787829718.909623:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000400:00000001:0.0:1787829718.909631:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787829718.909631:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829718.909634:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829718.909637:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829718.909640:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829718.909642:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00100000:1.0:1787829718.909642:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829718.909646:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829718.909686:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829718.909690:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829718.909696:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829718.909697:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829718.909727:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829718.909728:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829718.909733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829718.909737:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:1.0:1787829718.909738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829718.909740:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829718.909741:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829718.909743:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:1.0:1787829718.909745:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829718.909746:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829718.909752:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:0.0:1787829718.909753:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829718.909761:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3866
00000100:00000001:0.0:1787829718.909763:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829718.909771:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829718.909772:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829718.909781:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829718.909788:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829718.909791:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829718.909796:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829718.909796:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829718.909799:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.909801:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829718.909803:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829718.909804:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829718.909809:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1787829718.909811:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00080000:0.0:1787829718.909813:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787829718.909816:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00080000:0.0:1787829718.909821:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000020:00000001:1.0:1787829718.909822:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829718.909825:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829718.909827:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000001:1.0:1787829718.909829:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829718.909834:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000040:0.0:1787829718.909837:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829718.909839:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829718.909845:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829718.909854:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00100000:0.0:1787829718.909860:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327372160:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829718.909861:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829718.909867:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829718.909870:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829718.909871:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829718.909875:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:1.0:1787829718.909877:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000040:0.0:1787829718.909883:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327372160/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829718.909885:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829718.909893:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829718.909899:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829718.909900:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829718.909904:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:1.0:1787829718.909906:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1787829718.909907:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829718.909912:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.909914:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:1.0:1787829718.909916:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000020:00000001:0.0:1787829718.909918:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000008:00100000:1.0:1787829718.909921:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000020:00000040:0.0:1787829718.909922:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000008:00000001:1.0:1787829718.909926:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829718.909928:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00000001:1.0:1787829718.909931:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829718.909933:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829718.909939:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000010:1.0:1787829718.909939:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
02000000:00000001:0.0:1787829718.909943:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000040:1.0:1787829718.909945:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000010:0.0:1787829718.909947:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
02000000:00000001:1.0:1787829718.909950:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829718.909952:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
02000000:00000001:1.0:1787829718.909954:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909956:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829718.909958:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829718.909961:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829718.909962:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829718.909967:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909972:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:1.0:1787829718.909972:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:0.0:1787829718.909976:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829718.909979:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.909981:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829718.909984:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829718.909986:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00100000:1.0:1787829718.909989:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829718.909994:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829718.910012:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829718.910014:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829718.910018:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829718.910018:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.910024:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:1.0:1787829718.910026:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829718.910027:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829718.910033:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829718.910042:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372288:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829718.910046:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910049:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:1.0:1787829718.910052:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3876)
02000000:00000001:0.0:1787829718.910053:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829718.910057:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.910060:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829718.910064:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829718.910066:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.910070:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829718.910074:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829718.910078:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829718.910082:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372288, portal 10
00000100:00000001:0.0:1787829718.910084:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829718.910086:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829718.910094:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910102:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829718.910105:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829718.910107:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829718.910109:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327372288, offset 0
00000400:00000200:0.0:1787829718.910114:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829718.910118:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.910122:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829718.910126:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829718.910129:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829718.910132:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829718.910135:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829718.910138:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40680:40680:256:4294967295] 192.168.203.116@tcp LPNI seq info [40680:40680:8:4294967295]
00000400:00000200:0.0:1787829718.910143:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.910150:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829718.910155:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.910159:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829718.910163:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829718.910167:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829718.910171:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829718.910174:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829718.910179:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.910181:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829718.910184:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829718.910187:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829718.910191:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829718.910193:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829718.910196:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829718.910200:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910209:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829718.910212:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910220:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910221:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829718.910224:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829718.910228:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829718.910232:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829718.910234:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298539019
00000100:00000001:0.0:1787829718.910238:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298539692
00000100:00000001:0.0:1787829718.910240:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298539019
00000100:00000001:0.0:1787829718.910243:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298539692
00000100:00000001:0.0:1787829718.910245:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829718.910247:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910249:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829718.910253:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829718]  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829718.910261:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829718.910263:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829718.910264:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829718.910267:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829718.910268:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829718.910271:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829718.910274:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829718.910283:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910290:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829718.910292:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910294:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.910295:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829718.910299:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910307:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829718.910311:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910318:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829718.910320:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910323:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.910324:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910326:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829718.910329:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829718.910331:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829718.910335:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829718.910337:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829718.910342:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829718.910351:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372288:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829718.910355:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829718.910358:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910366:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829718.910367:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.910369:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829718.910371:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829718.910373:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829718.910376:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372416:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829718.910380:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829718.910382:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829718.910383:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.910385:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829718.910388:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829718.910390:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.910409:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829718.910413:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829718.910416:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829718.910420:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372416, portal 4
00000100:00000001:0.0:1787829718.910422:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829718.910424:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829718.910429:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910436:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829718.910439:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829718.910441:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829718.910444:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327372416, offset 0
00000400:00000200:0.0:1787829718.910448:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829718.910452:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.910456:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829718.910459:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829718.910462:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829718.910465:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829718.910467:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829718.910470:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40681:40681:256:4294967295] 192.168.203.116@tcp LPNI seq info [40681:40681:8:4294967295]
00000400:00000200:0.0:1787829718.910475:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.910481:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829718.910486:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.910489:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829718.910493:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829718.910496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829718.910500:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829718.910503:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829718.910507:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.910509:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829718.910512:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829718.910516:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829718.910519:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829718.910521:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829718.910523:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829718.910527:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910535:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829718.910539:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910546:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910548:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829718.910551:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829718.910554:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829718.910558:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829718.910560:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298539026
00000100:00000001:0.0:1787829718.910562:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298539692
00000100:00000001:0.0:1787829718.910564:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298539026
00000100:00000001:0.0:1787829718.910566:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298539692
00000100:00000001:0.0:1787829718.910568:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829718.910570:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910572:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829718.910577:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829718]  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829718.910585:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829718.910586:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829718.910588:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829718.910590:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829718.910592:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829718.910595:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829718.910598:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829718.910606:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910613:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829718.910615:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910616:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.910618:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829718.910622:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910629:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829718.910633:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910640:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829718.910643:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910645:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.910646:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910648:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829718.910649:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829718.910652:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829718.910677:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829718.910681:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829718.910686:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829718.910695:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372416:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829718.910699:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910701:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829718.910705:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327372288/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910712:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829718.910713:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829718.910715:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829718.910718:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829718.910719:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829718.910721:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829718.910723:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829718.910725:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829718.910729:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829718.910730:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829718.910732:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829718.910734:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829718.910736:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910737:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829718.910741:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327372416/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910748:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829718.910749:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829718.910751:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829718.910753:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829718.910754:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829718.910756:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829718.910758:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829718.910759:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829718.910762:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829718.910763:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829718.910765:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829718.910767:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829718.910769:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910771:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.910774:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829718.910776:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910778:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829718.910779:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829718.910781:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829718.910785:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910791:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829718.910793:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.910795:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829718.910797:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829718.910799:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829718.910802:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372544:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829718.910806:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829718.910808:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829718.910809:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829718.910811:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829718.910814:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829718.910816:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829718.910819:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829718.910822:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829718.910826:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829718.910829:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372544, portal 4
00000100:00000001:0.0:1787829718.910831:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829718.910832:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:0.0:1787829718.910838:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910844:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829718.910847:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829718.910849:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829718.910852:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327372544, offset 0
00000400:00000200:0.0:1787829718.910856:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829718.910860:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.910863:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829718.910866:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829718.910870:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829718.910872:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829718.910875:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829718.910878:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40682:40682:256:4294967295] 192.168.203.116@tcp LPNI seq info [40682:40682:8:4294967295]
00000400:00000200:0.0:1787829718.910882:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829718.910888:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829718.910893:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.910896:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829718.910901:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829718.910904:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829718.910908:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829718.910910:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829718.910914:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829718.910916:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829718.910919:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829718.910922:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829718.910926:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829718.910928:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829718.910930:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829718.910934:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910942:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829718.910945:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.910952:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910954:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829718.910957:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829718.910960:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829718.910963:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829718.910965:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298539032
00000100:00000001:0.0:1787829718.910968:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298539692
00000100:00000001:0.0:1787829718.910970:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298539032
00000100:00000001:0.0:1787829718.910973:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298539692
00000100:00000001:0.0:1787829718.910975:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829718.910977:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.910979:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829718.910983:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829718]  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829718.910990:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829718.910992:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829718.911017:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829718.911020:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829718.911022:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829718.911025:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829718.911028:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829718.911038:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.911044:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829718.911046:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.911048:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.911049:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829718.911053:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.911060:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829718.911064:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.911072:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829718.911074:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.911076:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829718.911077:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.911079:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829718.911081:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829718.911084:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829718.911087:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829718.911089:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829718.911093:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829718.911101:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372544:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829718.911105:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.911107:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829718.911110:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327372544/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829718.911117:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829718.911119:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829718.911121:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829718.911123:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829718.911125:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829718.911127:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829718.911129:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829718.911130:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829718.911133:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829718.911134:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829718.911136:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829718.911138:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829718.911139:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.911141:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829718.911144:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829718.911145:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829718.911147:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829718.911149:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829719.420761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829719.420768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829719.420769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829719.420773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829719.420776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829719.420779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829719.420781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829719.420789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829719.420791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829719.612751:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298539692, at 4298539712
00000100:00000040:3.0:1787829719.612751:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298539692, at 4298539712
00000100:00000010:3.0:1787829719.612768:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00000100:00000010:2.0:1787829719.612775:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00010000:00000001:0.0:1787829719.740784:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829719.740792:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829719.740799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829719.740801:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829719.740805:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829719.740808:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829719.740811:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829719.740815:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829719.740818:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829719.740821:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829719.740823:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829719.740825:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829719.740864:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829719.740868:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829719.740870:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829720.444761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829720.444767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829720.444769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829720.444773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829720.444776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829720.444779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829720.444780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829720.444782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829720.444784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829720.764752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829720.764760:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829720.764766:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829720.764768:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829720.764771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829720.764773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829720.764793:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829720.764796:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829720.764800:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829721.468798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829721.468806:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829721.468808:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829721.468814:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829721.468817:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829721.468821:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829721.468824:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829721.468826:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829721.468829:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829721.788759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829721.788765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829721.788776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829721.788778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829721.788781:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829721.788782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829721.788843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829721.788848:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829721.788852:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829722.492781:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829722.492788:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829722.492789:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829722.492793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829722.492804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829722.492807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829722.492809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829722.492812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829722.492814:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829722.812765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829722.812773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829722.812779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829722.812781:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829722.812785:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829722.812786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829722.812829:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829722.812833:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829722.812836:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829723.516754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829723.516759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829723.516761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829723.516765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829723.516768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829723.516770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829723.516772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829723.516774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829723.516776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829723.836757:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829723.836764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829723.836769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829723.836770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829723.836772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829723.836774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829723.836791:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829723.836793:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829723.836795:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829724.028791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829724.028802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829724.028806:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829724.028810:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829724.028815:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829724.028822:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3871
00000400:00000200:0.0:1787829724.028831:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829724.028838:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829724.028842:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829724.028845:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.028849:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829724.028851:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.028854:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829724.028858:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829724.028860:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829724.028863:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829724.028866:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829724.028869:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829724.028871:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829724.028874:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.028877:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829724.028887:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.028889:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829724.028893:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829724.028897:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829724.028900:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829724.028903:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829724.028906:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829724.028909:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.028911:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829724.028914:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829724.028923:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716de00.
00000020:00000040:0.0:1787829724.028926:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829724.028930:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829724.028932:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.028935:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829724.028938:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829724.028942:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829724.028949:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533000.
00000100:00000001:0.0:1787829724.028955:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829724.028964:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829724.028975:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829724.028980:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829724.028985:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c17e5d1d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829724.029007:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829724.029010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829724.029013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829724.029015:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829724.029017:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829724.029021:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829724.029024:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3871
00000400:00000200:0.0:1787829724.029028:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829724.029030:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787829724.029034:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829724.029037:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829724.029041:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829724.029042:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829724.029044:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787829724.029047:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829724.029048:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029053:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829724.029056:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829724.029059:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000040:2.0:1787829724.029060:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787829724.029061:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829724.029065:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829724.029069:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:0.0:1787829724.029074:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:2.0:1787829724.029082:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829724.029086:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829724.029090:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:2.0:1787829724.029094:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787829724.029095:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829724.029098:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029100:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:2.0:1787829724.029104:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787829724.029105:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:2.0:1787829724.029110:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829724.029111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029116:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829724.029118:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327372672:192.168.203.116@tcp:250:kworker.0
02000000:00000001:0.0:1787829724.029121:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787829724.029125:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829724.029129:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829724.029129:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829724.029133:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829724.029135:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.029137:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829724.029139:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787829724.029143:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
00000008:00100000:0.0:1787829724.029145:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787829724.029147:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829724.029151:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029155:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829724.029156:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829724.029163:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000100:00000010:0.0:1787829724.029164:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716ed00.
00000020:00000040:0.0:1787829724.029168:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:2.0:1787829724.029169:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
02000000:00000001:0.0:1787829724.029172:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829724.029175:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029178:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1787829724.029178:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372672, portal 25
02000000:00000001:0.0:1787829724.029181:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787829724.029184:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787829724.029185:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029189:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
02000000:00000010:0.0:1787829724.029192:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530400.
00000100:00000001:0.0:1787829724.029198:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.029201:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829724.029207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829724.029226:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787829724.029228:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829724.029232:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.029233:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829724.029239:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000040:0.0:1787829724.029241:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008f0359f0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829724.029243:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327372672, offset 0
00000400:00000200:2.0:1787829724.029256:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829724.029267:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829724.029269:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829724.029274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787829724.029276:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:0.0:1787829724.029280:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829724.029283:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787829724.029283:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1787829724.029286:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787829724.029289:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:0.0:1787829724.029292:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787829724.029295:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1787829724.029299:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3871
00000400:00000200:2.0:1787829724.029300:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829724.029310:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.029310:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40683:40683:256:4294967295] 192.168.203.116@tcp LPNI seq info [40683:40683:8:4294967295]
00000400:00000200:0.0:1787829724.029319:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:2.0:1787829724.029320:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829724.029325:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829724.029331:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029336:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787829724.029337:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787829724.029342:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029345:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:2.0:1787829724.029347:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:0.0:1787829724.029349:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829724.029352:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000010:2.0:1787829724.029353:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000020:00000001:0.0:1787829724.029355:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829724.029372:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:0.0:1787829724.029378:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:2.0:1787829724.029378:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:0.0:1787829724.029384:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:2.0:1787829724.029385:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000100:00000040:0.0:1787829724.029392:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:2.1:1787829724.029395:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829724.029397:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787829724.029400:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000100:00000040:0.0:1787829724.029403:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029409:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000100:2.0:1787829724.029409:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.029414:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000020:00000040:0.0:1787829724.029415:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:2.0:1787829724.029421:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787829724.029422:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029427:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829724.029428:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787829724.029433:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787829724.029435:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
02000000:00000001:0.0:1787829724.029438:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1787829724.029439:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
02000000:00000001:0.0:1787829724.029445:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029447:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829724.029450:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1787829724.029454:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787829724.029455:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829724.029462:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029466:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829724.029473:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f480.
00000100:00000001:2.0:1787829724.029474:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787829724.029480:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000040:2.0:1787829724.029480:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829724.029487:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829724.029491:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029492:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029494:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787829724.029495:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1787829724.029499:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787829724.029502:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829724.029503:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829724.029511:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:0.0:1787829724.029512:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531400.
00000100:00000001:0.0:1787829724.029516:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029522:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298544812
00000100:00100000:0.0:1787829724.029523:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1787829724.029530:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee95a0.
00000100:00000001:0.0:1787829724.029538:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829724.029540:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829724.029541:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029547:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298544812
00000100:00000040:0.0:1787829724.029550:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84d4029] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787829724.029552:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee9780.
00000100:00000001:2.0:1787829724.029557:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829724.029566:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829724.029569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787829724.029570:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787829724.029575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829724.029575:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029580:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787829724.029581:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1787829724.029585:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787829724.029589:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829724/real 1787829724]  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829724.029591:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829724.029598:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3871
00000400:00000200:0.0:1787829724.029608:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829724.029615:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829724.029617:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000001:2.0:1787829724.029623:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1787829724.029624:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000001:2.0:1787829724.029629:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787829724.029630:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829724.029635:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:2.0:1787829724.029635:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000100:00000001:0.0:1787829724.029642:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:2.0:1787829724.029643:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000020:00000001:0.0:1787829724.029645:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829724.029649:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000010:2.0:1787829724.029650:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000020:00000001:0.0:1787829724.029651:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787829724.029690:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:0.0:1787829724.029693:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:0.0:1787829724.029699:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:2.0:1787829724.029701:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787829724.029703:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:0.0:1787829724.029712:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029717:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829724.029722:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029726:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:2.0:1787829724.029728:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829724.029729:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829724.029737:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829724.029743:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829724.029747:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787829724.029748:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1787829724.029750:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829724.029753:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829724.029755:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029759:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829724.029759:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.029764:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1787829724.029765:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829724.029771:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029775:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829724.029777:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1787829724.029780:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716d2c0.
00000020:00000040:0.0:1787829724.029784:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:0.0:1787829724.029791:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829724.029793:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.029798:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787829724.029800:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829724.029801:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829724.029805:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.029810:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829724.029811:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532400.
00000100:00000001:0.0:1787829724.029817:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829724.029824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829724.029851:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829724.029853:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829724.029856:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029861:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829724.029863:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c4c777e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.029867:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.029870:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.029876:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829724.029881:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829724.029886:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3881)
00000100:00000001:2.0:1787829724.029889:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829724.029895:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829724.029901:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829724.029904:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829724.029911:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829724.029922:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327372672:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787829724.029927:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.029929:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.029933:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327372672/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.029940:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829724.029942:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829724.029945:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829724.029949:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829724.029950:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829724.029952:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787829724.029954:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829724.029957:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:2.0:1787829724.029960:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829724.029962:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829724.029964:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:2.0:1787829724.029967:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829724.029969:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.029971:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.029977:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829724.029979:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.029982:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829724.029983:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829724.029985:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829724.029988:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.029996:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829724.029997:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.030000:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829724.030001:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829724.030004:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829724.030007:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372800:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829724.030010:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829724.030013:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829724.030014:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.030016:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829724.030020:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829724.030022:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829724.030026:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829724.030029:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829724.030033:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829724.030036:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372800, portal 10
00000100:00000001:2.0:1787829724.030038:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829724.030039:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:2.0:1787829724.030047:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030054:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829724.030056:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829724.030059:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829724.030061:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327372800, offset 0
00000400:00000200:2.0:1787829724.030066:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829724.030070:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.030074:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829724.030077:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829724.030081:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829724.030083:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829724.030086:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829724.030089:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40684:40684:256:4294967295] 192.168.203.116@tcp LPNI seq info [40684:40684:8:4294967295]
00000400:00000200:2.0:1787829724.030094:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.030100:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829724.030105:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.030109:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829724.030113:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829724.030117:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787829724.030122:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829724.030125:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787829724.030129:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.030131:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829724.030134:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829724.030138:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829724.030141:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829724.030143:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829724.030146:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829724.030150:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030158:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.030161:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030167:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.030169:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829724.030172:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.030176:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787829724.030179:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829724.030182:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298544139
00000100:00000001:2.0:1787829724.030185:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298544812
00000100:00000001:2.0:1787829724.030187:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298544139
00000100:00000001:2.0:1787829724.030189:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298544812
00000100:00000001:2.0:1787829724.030191:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829724.030194:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.030195:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829724.030200:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829724/real 1787829724]  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829724.030208:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829724.030209:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829724.030212:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829724.030215:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829724.030217:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829724.030220:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829724.030223:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829724.030233:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030240:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829724.030242:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030243:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.030245:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829724.030249:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030256:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829724.030260:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030267:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829724.030270:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030271:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.030273:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030274:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829724.030277:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829724.030280:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829724.030283:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829724.030286:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829724.030290:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829724.030299:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372800:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829724.030304:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829724.030308:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030315:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829724.030317:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.030319:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829724.030321:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829724.030323:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829724.030327:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372928:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829724.030330:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829724.030333:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829724.030334:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.030336:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829724.030340:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829724.030342:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829724.030346:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829724.030348:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829724.030352:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829724.030355:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327372928, portal 4
00000100:00000001:2.0:1787829724.030357:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829724.030375:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:2.0:1787829724.030382:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030390:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829724.030393:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829724.030396:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829724.030399:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327372928, offset 0
00000400:00000200:2.0:1787829724.030404:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829724.030408:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.030412:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829724.030416:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829724.030419:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829724.030421:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829724.030424:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829724.030427:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40685:40685:256:4294967295] 192.168.203.116@tcp LPNI seq info [40685:40685:8:4294967295]
00000400:00000200:2.0:1787829724.030431:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.030438:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829724.030442:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.030446:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829724.030450:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829724.030453:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787829724.030458:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829724.030461:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787829724.030465:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.030467:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829724.030470:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829724.030474:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829724.030477:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829724.030479:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829724.030482:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829724.030486:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030495:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.030498:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030504:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.030506:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829724.030509:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.030513:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787829724.030516:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829724.030518:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298544146
00000100:00000001:2.0:1787829724.030521:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298544812
00000100:00000001:2.0:1787829724.030523:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298544146
00000100:00000001:2.0:1787829724.030526:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298544812
00000100:00000001:2.0:1787829724.030528:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829724.030530:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.030532:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829724.030537:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829724/real 1787829724]  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829724.030545:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829724.030547:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829724.030549:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829724.030551:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829724.030553:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829724.030555:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829724.030558:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829724.030568:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030575:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829724.030577:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030579:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.030580:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829724.030584:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030592:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829724.030596:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030603:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829724.030605:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030607:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.030608:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030611:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829724.030612:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829724.030615:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829724.030618:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829724.030621:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829724.030625:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829724.030634:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327372928:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829724.030639:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829724.030642:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030649:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829724.030651:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.030653:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829724.030674:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829724.030677:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829724.030680:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373056:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829724.030684:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829724.030686:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829724.030688:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829724.030689:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829724.030693:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787829724.030695:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829724.030699:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829724.030701:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829724.030705:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829724.030708:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373056, portal 4
00000100:00000001:2.0:1787829724.030710:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829724.030712:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:2.0:1787829724.030717:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030725:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829724.030727:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829724.030730:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829724.030732:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327373056, offset 0
00000400:00000200:2.0:1787829724.030736:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829724.030741:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.030744:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829724.030747:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829724.030750:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829724.030752:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829724.030755:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829724.030758:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40686:40686:256:4294967295] 192.168.203.116@tcp LPNI seq info [40686:40686:8:4294967295]
00000400:00000200:2.0:1787829724.030762:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829724.030769:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829724.030774:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.030777:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829724.030781:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829724.030784:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787829724.030788:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829724.030791:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787829724.030795:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829724.030798:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829724.030801:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829724.030805:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829724.030808:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829724.030810:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829724.030812:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829724.030816:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030824:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.030828:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829749 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030841:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.030843:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829724.030846:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.030849:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787829724.030853:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829724.030855:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298544152
00000100:00000001:2.0:1787829724.030857:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298544812
00000100:00000001:2.0:1787829724.030859:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298544152
00000100:00000001:2.0:1787829724.030861:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298544812
00000100:00000001:2.0:1787829724.030863:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829724.030865:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.030867:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829724.030871:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829724/real 1787829724]  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829724.030878:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829724.030879:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829724.030881:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829724.030883:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829724.030884:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829724.030887:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829724.030889:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829724.030897:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030903:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829724.030904:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030906:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.030907:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829724.030911:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030919:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829724.030923:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030931:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829724.030933:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030935:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829724.030936:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030938:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829724.030940:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829724.030943:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829724.030946:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829724.030948:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829724.030953:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829724.030962:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373056:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829724.030966:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.030968:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.030971:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327372800/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.030978:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829724.030980:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829724.030982:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829724.030984:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829724.030986:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829724.030988:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787829724.030991:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829724.030992:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:2.0:1787829724.030995:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829724.030997:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829724.030998:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:2.0:1787829724.031001:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829724.031002:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.031004:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.031008:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327372928/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.031015:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829724.031017:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829724.031018:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829724.031021:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829724.031022:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829724.031024:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787829724.031026:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829724.031028:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:2.0:1787829724.031030:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829724.031032:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829724.031033:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:2.0:1787829724.031036:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829724.031037:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.031039:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829724.031043:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327373056/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829749 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829724.031050:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829724.031051:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829724.031053:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787829724.031055:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829724.031056:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829724.031058:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787829724.031060:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829724.031061:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:2.0:1787829724.031064:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829724.031065:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829724.031066:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:2.0:1787829724.031068:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829724.031070:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.031072:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829724.031076:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829724.031077:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829724.031079:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829724.031081:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829724.540757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829724.540764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829724.540766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829724.540769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829724.540772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829724.540775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829724.540777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829724.540779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829724.540781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829724.732781:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298544812, at 4298544832
00000100:00000040:3.0:1787829724.732781:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298544812, at 4298544832
00000100:00000010:3.0:1787829724.732800:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee9780.
00000100:00000010:2.0:1787829724.732813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee95a0.
00010000:00000001:0.0:1787829724.860777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829724.860786:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829724.860788:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829724.860791:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860803:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829724.860805:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860807:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860809:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860813:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829724.860814:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829724.860815:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829724.860816:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860818:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829724.860819:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860820:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860821:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829724.860824:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829724.860826:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829724.860878:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829724.860883:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829724.860887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829725.564783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829725.564794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829725.564797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829725.564804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829725.564809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829725.564813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829725.564817:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829725.564820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829725.564824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829725.884756:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829725.884763:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829725.884768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829725.884769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829725.884772:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829725.884773:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829725.884828:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829725.884834:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829725.884838:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829726.588850:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829726.588856:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829726.588858:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829726.588870:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829726.588873:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829726.588877:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829726.588886:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829726.588889:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829726.588892:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829726.908763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829726.908769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829726.908773:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829726.908774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829726.908776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829726.908777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829726.908798:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829726.908800:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829726.908802:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829727.612736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829727.612741:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829727.612743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829727.612747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829727.612750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829727.612753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829727.612755:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829727.612757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829727.612759:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829727.932767:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829727.932772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829727.932777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829727.932778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829727.932780:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829727.932781:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829727.932850:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829727.932853:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829727.932856:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829728.636759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829728.636765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829728.636766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829728.636770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829728.636772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829728.636775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829728.636777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829728.636779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829728.636782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829728.956744:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829728.956750:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829728.956754:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829728.956755:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829728.956758:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829728.956759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829728.956794:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829728.956798:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829728.956801:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829729.148804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829729.148817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829729.148820:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829729.148824:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829729.148828:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829729.148836:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3876
00000400:00000200:3.0:1787829729.148845:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829729.148851:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829729.148855:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829729.148858:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.148862:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829729.148864:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.148867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829729.148870:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829729.148873:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829729.148875:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829729.148879:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829729.148881:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829729.148884:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829729.148887:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.148889:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829729.148892:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.148894:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829729.148897:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829729.148900:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829729.148903:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829729.148906:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829729.148910:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829729.148912:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.148915:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829729.148918:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829729.148926:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829729.148930:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829729.148934:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829729.148936:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.148939:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829729.148942:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829729.148946:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829729.148961:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829729.148969:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829729.148978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829729.148988:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829729.148994:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829729.148999:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829729.149021:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829729.149025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829729.149029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829729.149031:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829729.149032:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829729.149035:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829729.149039:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3876
00000400:00000200:3.0:1787829729.149044:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829729.149049:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829729.149051:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829729.149055:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829729.149058:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787829729.149058:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787829729.149064:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787829729.149064:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829729.149067:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829729.149069:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829729.149074:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829729.149075:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829729.149077:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829729.149079:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000040:2.0:1787829729.149082:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829729.149083:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829729.149087:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829729.149098:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829729.149099:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829729.149102:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149104:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787829729.149107:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787829729.149110:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:2.0:1787829729.149112:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829729.149117:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:2.0:1787829729.149118:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787829729.149125:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00100000:2.0:1787829729.149129:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327373184:192.168.203.116@tcp:250:kworker.0
00000100:00080000:3.0:1787829729.149133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829729.149138:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787829729.149139:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.149144:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787829729.149144:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829729.149149:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829729.149150:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829729.149153:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787829729.149155:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149159:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787829729.149161:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
00000008:00100000:3.0:1787829729.149164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787829729.149166:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787829729.149170:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149174:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829729.149177:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:3.0:1787829729.149180:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000400:00000010:2.0:1787829729.149184:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000020:00000040:3.0:1787829729.149185:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829729.149191:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829729.149194:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787829729.149195:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000001:3.0:1787829729.149196:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829729.149199:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829729.149202:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787829729.149205:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373184, portal 25
02000000:00000010:3.0:1787829729.149210:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:2.0:1787829729.149211:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829729.149216:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000001:3.0:1787829729.149217:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829729.149225:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787829729.149228:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829729.149248:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787829729.149253:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787829729.149256:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149259:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787829729.149262:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000040:3.0:1787829729.149268:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829729.149269:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327373184, offset 0
00000400:00000200:2.0:1787829729.149282:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149287:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829729.149292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787829729.149294:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829729.149300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1787829729.149304:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829729.149305:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829729.149309:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787829729.149312:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787829729.149315:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787829729.149318:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829729.149336:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1787829729.149336:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3876
00000400:00000200:2.0:1787829729.149341:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829729.149346:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40687:40687:256:4294967295] 192.168.203.116@tcp LPNI seq info [40687:40687:8:4294967295]
00000400:00000200:2.0:1787829729.149350:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.149358:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829729.149362:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:3.0:1787829729.149364:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000010:2.0:1787829729.149368:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000400:00000200:3.0:1787829729.149369:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829729.149374:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1787829729.149380:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829729.149382:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:2.0:1787829729.149386:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000100:00000001:3.0:1787829729.149387:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829729.149391:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149395:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829729.149397:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829729.149399:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829729.149401:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1787829729.149402:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:3.0:1787829729.149404:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829729.149408:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:2.1:1787829729.149409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000020:00000001:3.0:1787829729.149411:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000100:2.0:1787829729.149420:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000040:3.0:1787829729.149421:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149426:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1787829729.149427:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000100:00000040:3.0:1787829729.149430:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149435:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:2.0:1787829729.149437:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:3.0:1787829729.149439:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:2.0:1787829729.149447:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787829729.149447:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149455:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829729.149456:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
02000000:00000001:3.0:1787829729.149459:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1787829729.149460:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
02000000:00000001:3.0:1787829729.149466:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829729.149467:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787829729.149472:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787829729.149474:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829729.149476:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829729.149481:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829729.149488:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149493:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829729.149500:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000100:00000001:2.0:1787829729.149504:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:3.0:1787829729.149506:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000040:2.0:1787829729.149512:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829729.149513:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829729.149516:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149521:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829729.149524:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829729.149528:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149530:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149534:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:3.0:1787829729.149535:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:2.0:1787829729.149542:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149542:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829729.149552:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:3.0:1787829729.149555:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829729.149557:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298549932
00000100:00000010:2.0:1787829729.149567:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee95a0.
00000100:00000001:3.0:1787829729.149568:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829729.149573:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149578:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787829729.149579:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.149588:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298549932
00000100:00000001:3.0:1787829729.149592:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829729.149595:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee92a0.
00000100:00080000:3.0:1787829729.149597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787829729.149601:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787829729.149603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829729.149608:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829729.149608:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829729.149611:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829729.149614:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829729.149617:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787829729.149620:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1787829729.149623:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3876
00000100:00100000:2.0:1787829729.149629:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829729/real 1787829729]  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787829729.149630:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829729.149638:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829729.149644:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829729.149651:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000001:2.0:1787829729.149652:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787829729.149686:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829729.149691:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000001:2.0:1787829729.149692:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787829729.149695:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:2.0:1787829729.149697:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000020:00000001:3.0:1787829729.149701:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787829729.149703:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000020:00000040:3.0:1787829729.149705:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000010:2.0:1787829729.149707:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000020:00000001:3.0:1787829729.149709:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787829729.149712:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:3.0:1787829729.149714:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000200:2.0:1787829729.149720:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829729.149721:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829729.149729:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829729.149738:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:2.0:1787829729.149741:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829729.149743:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829729.149750:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829729.149754:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787829729.149755:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:2.0:1787829729.149757:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.149762:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1787829729.149762:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:2.0:1787829729.149766:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787829729.149770:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:2.0:1787829729.149774:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829729.149776:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829729.149782:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829729.149788:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829729.149790:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787829729.149792:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829729.149798:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829729.149798:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829729.149803:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829729.149810:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149812:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787829729.149816:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149817:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.149822:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.149823:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.149825:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000010:3.0:1787829729.149825:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787829729.149828:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829729.149832:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:3.0:1787829729.149832:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00080000:2.0:1787829729.149838:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
02000000:00000001:3.0:1787829729.149840:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787829729.149841:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:3.0:1787829729.149845:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829729.149847:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829729.149851:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829729.149855:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:2.0:1787829729.149858:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327373184:192.168.203.116@tcp:250:kworker.0
02000000:00000001:3.0:1787829729.149860:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149862:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.149866:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.149869:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327373184/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787829729.149869:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829729.149877:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149882:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829729.149887:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00100000:3.0:1787829729.149888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:2.0:1787829729.149891:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829729.149897:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829729.149900:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000001:3.0:1787829729.149903:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000020:00000040:2.0:1787829729.149904:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000100:00000001:3.0:1787829729.149910:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829729.149911:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829729.149915:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
00000100:00000040:3.0:1787829729.149921:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829729.149923:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829729.149927:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829729.149930:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:2.0:1787829729.149936:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787829729.149938:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149939:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.149944:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829729.149946:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3886)
00000100:00000001:2.0:1787829729.149952:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829729.149956:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.149959:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829729.149961:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829729.149962:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829729.149969:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.149977:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829729.149979:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.149981:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829729.149983:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829729.149985:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829729.149989:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373312:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829729.149992:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829729.149994:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829729.149996:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.149998:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829729.150002:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829729.150004:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829729.150008:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829729.150011:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829729.150015:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829729.150018:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373312, portal 10
00000100:00000001:2.0:1787829729.150020:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829729.150022:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:2.0:1787829729.150029:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150037:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829729.150040:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829729.150042:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829729.150045:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327373312, offset 0
00000400:00000200:2.0:1787829729.150050:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829729.150054:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.150058:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829729.150062:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829729.150065:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829729.150068:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829729.150070:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829729.150074:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40688:40688:256:4294967295] 192.168.203.116@tcp LPNI seq info [40688:40688:8:4294967295]
00000400:00000200:2.0:1787829729.150079:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.150086:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829729.150091:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829729.150095:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829729.150099:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829729.150103:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787829729.150107:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829729.150110:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000100:2.0:1787829729.150115:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829729.150117:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829729.150120:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829729.150124:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829729.150127:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829729.150129:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829729.150132:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829729.150136:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150145:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.150148:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150163:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.150165:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829729.150168:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829729.150172:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787829729.150176:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829729.150178:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298549259
00000100:00000001:2.0:1787829729.150181:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298549932
00000100:00000001:2.0:1787829729.150183:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298549259
00000100:00000001:2.0:1787829729.150186:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298549932
00000100:00000001:2.0:1787829729.150188:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829729.150190:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.150192:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829729.150196:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829729/real 1787829729]  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829729.150204:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829729.150206:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829729.150208:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829729.150210:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829729.150211:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829729.150214:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829729.150218:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829729.150226:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150234:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829729.150235:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150237:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.150238:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829729.150242:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150250:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829729.150254:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150262:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829729.150264:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150266:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.150267:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150270:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829729.150272:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829729.150275:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829729.150279:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829729.150282:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829729.150286:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829729.150295:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373312:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829729.150299:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829729.150303:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150310:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829729.150312:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.150314:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829729.150316:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829729.150319:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829729.150322:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373440:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829729.150341:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829729.150343:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829729.150346:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.150348:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829729.150352:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829729.150354:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829729.150358:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829729.150361:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829729.150365:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829729.150368:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373440, portal 4
00000100:00000001:2.0:1787829729.150370:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829729.150372:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:2.0:1787829729.150378:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150385:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829729.150388:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829729.150390:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829729.150393:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327373440, offset 0
00000400:00000200:2.0:1787829729.150397:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829729.150401:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.150405:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829729.150408:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829729.150411:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829729.150414:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829729.150416:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829729.150420:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40689:40689:256:4294967295] 192.168.203.116@tcp LPNI seq info [40689:40689:8:4294967295]
00000400:00000200:2.0:1787829729.150424:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.150431:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829729.150436:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829729.150440:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829729.150444:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829729.150447:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787829729.150452:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829729.150455:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000100:2.0:1787829729.150460:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829729.150462:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829729.150465:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829729.150469:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829729.150472:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829729.150474:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829729.150476:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829729.150480:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150489:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.150493:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150500:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.150502:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829729.150505:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829729.150508:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787829729.150511:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829729.150513:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298549266
00000100:00000001:2.0:1787829729.150516:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298549932
00000100:00000001:2.0:1787829729.150518:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298549266
00000100:00000001:2.0:1787829729.150521:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298549932
00000100:00000001:2.0:1787829729.150523:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829729.150525:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.150527:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829729.150531:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829729/real 1787829729]  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829729.150539:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829729.150541:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829729.150543:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829729.150545:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829729.150547:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829729.150549:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829729.150553:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829729.150563:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150570:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829729.150571:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150573:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.150574:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829729.150578:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150586:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829729.150590:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150598:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829729.150600:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150602:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.150603:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150605:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829729.150607:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829729.150610:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829729.150613:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829729.150615:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829729.150619:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829729.150626:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373440:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829729.150630:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829729.150633:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150641:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829729.150642:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.150644:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829729.150646:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829729.150648:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829729.150651:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373568:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829729.150654:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829729.150674:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829729.150676:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829729.150678:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829729.150682:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787829729.150685:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829729.150688:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829729.150691:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829729.150694:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829729.150698:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373568, portal 4
00000100:00000001:2.0:1787829729.150699:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829729.150701:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:2.0:1787829729.150706:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150713:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829729.150716:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829729.150718:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829729.150721:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327373568, offset 0
00000400:00000200:2.0:1787829729.150725:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829729.150729:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.150733:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829729.150736:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829729.150739:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829729.150741:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829729.150744:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829729.150747:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40690:40690:256:4294967295] 192.168.203.116@tcp LPNI seq info [40690:40690:8:4294967295]
00000400:00000200:2.0:1787829729.150752:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829729.150759:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829729.150764:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829729.150768:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829729.150772:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829729.150775:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787829729.150779:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829729.150782:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 23 secs later)
00000800:00000100:2.0:1787829729.150799:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829729.150801:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829729.150805:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829729.150809:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829729.150813:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829729.150815:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829729.150818:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829729.150822:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150830:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.150834:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150842:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.150843:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829729.150847:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829729.150850:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787829729.150853:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829729.150856:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298549272
00000100:00000001:2.0:1787829729.150858:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298549932
00000100:00000001:2.0:1787829729.150861:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298549272
00000100:00000001:2.0:1787829729.150863:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298549932
00000100:00000001:2.0:1787829729.150865:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829729.150868:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.150870:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829729.150874:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829729/real 1787829729]  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829729.150883:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829729.150884:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829729.150886:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829729.150888:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829729.150890:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829729.150892:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829729.150896:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829729.150905:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150912:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829729.150913:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150915:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.150916:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829729.150920:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150927:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829729.150931:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150939:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829729.150941:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150943:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829729.150945:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150946:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829729.150948:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829729.150951:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829729.150955:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829729.150957:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829729.150962:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829729.150971:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373568:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829729.150974:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.150976:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.150980:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327373312/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.150987:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829729.150988:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829729.150990:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829729.150993:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829729.150994:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829729.150996:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787829729.150998:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829729.151000:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:2.0:1787829729.151002:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829729.151003:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829729.151005:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:2.0:1787829729.151008:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829729.151009:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.151011:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.151014:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327373440/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.151021:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829729.151022:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829729.151024:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829729.151026:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829729.151027:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829729.151028:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787829729.151029:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829729.151031:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:2.0:1787829729.151033:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829729.151035:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829729.151036:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:2.0:1787829729.151040:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829729.151041:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.151043:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829729.151046:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327373568/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829729.151053:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829729.151055:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829729.151056:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002400.
02000000:00000001:2.0:1787829729.151059:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829729.151060:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829729.151061:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787829729.151063:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829729.151065:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:2.0:1787829729.151067:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829729.151068:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829729.151070:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787829729.151072:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829729.151074:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.151076:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829729.151080:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829729.151082:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829729.151084:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829729.151086:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829729.660729:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829729.660734:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829729.660736:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829729.660740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829729.660743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829729.660752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829729.660754:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829729.660756:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829729.660758:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829729.852724:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298549932, at 4298549952
00000100:00000010:3.0:1787829729.852742:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee92a0.
00000100:00000040:2.0:1787829729.852768:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298549932, at 4298549952
00000100:00000010:2.0:1787829729.852782:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee95a0.
00010000:00000001:0.0:1787829729.980779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829729.980786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829729.980791:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829729.980793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829729.980796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829729.980799:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829729.980802:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829729.980804:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829729.980807:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829729.980810:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829729.980812:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829729.980814:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829729.980887:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829729.980891:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829729.980896:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829730.684757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829730.684762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829730.684764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829730.684769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829730.684772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829730.684774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829730.684776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829730.684778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829730.684780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829731.004752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829731.004759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829731.004765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829731.004766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829731.004770:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829731.004772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829731.004812:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829731.004815:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829731.004818:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829731.708741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829731.708746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829731.708748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829731.708752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829731.708755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829731.708757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829731.708759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829731.708762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829731.708765:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829732.028775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829732.028780:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829732.028792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829732.028794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829732.028797:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829732.028798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829732.028833:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829732.028838:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829732.028841:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829732.732760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829732.732766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829732.732769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829732.732774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829732.732777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829732.732781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829732.732784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829732.732786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829732.732789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829733.052765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829733.052770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829733.052775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829733.052776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829733.052779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829733.052780:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829733.052796:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829733.052798:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829733.052800:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829733.756721:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829733.756726:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829733.756728:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829733.756732:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829733.756735:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829733.756738:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829733.756740:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829733.756742:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829733.756744:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829734.076765:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829734.076772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829734.076778:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829734.076779:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829734.076783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829734.076784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829734.076848:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829734.076853:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829734.076856:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829734.268731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829734.268740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829734.268742:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829734.268745:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829734.268749:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829734.268756:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3881
00000400:00000200:3.0:1787829734.268764:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829734.268771:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829734.268773:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829734.268776:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268778:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829734.268780:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268782:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829734.268784:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829734.268786:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829734.268788:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829734.268790:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829734.268792:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829734.268794:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829734.268796:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268797:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829734.268799:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268800:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829734.268802:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829734.268804:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268806:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829734.268808:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829734.268811:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829734.268813:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.268814:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829734.268816:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829734.268824:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829734.268826:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829734.268829:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829734.268830:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.268832:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829734.268834:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829734.268837:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829734.268843:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829734.268848:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829734.268855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829734.268863:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829734.268866:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829734.268870:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829734.268888:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829734.268890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829734.268892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829734.268894:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829734.268895:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829734.268899:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829734.268903:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3881
00000400:00000200:3.0:1787829734.268907:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829734.268909:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1787829734.268912:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:2.0:1787829734.268914:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787829734.268916:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787829734.268918:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787829734.268924:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:2.0:1787829734.268927:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829734.268930:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829734.268936:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829734.268939:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829734.268942:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.268942:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:2.0:1787829734.268947:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:3.0:1787829734.268948:0:36:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787829734.268951:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1787829734.268951:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829734.268955:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787829734.268956:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787829734.268958:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00100000:2.0:1787829734.268961:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327373696:192.168.203.116@tcp:250:kworker.0
00000020:00000001:3.0:1787829734.268962:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829734.268968:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:2.0:1787829734.268969:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1787829734.268973:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268975:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787829734.268977:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:3.0:1787829734.268978:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:2.0:1787829734.268981:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.268982:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:2.0:1787829734.268988:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1787829734.268988:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829734.268990:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829734.268993:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829734.268994:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1787829734.268997:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000400.
02000000:00000001:3.0:1787829734.268999:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829734.269001:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829734.269003:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.269006:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829734.269010:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787829734.269011:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00000001:3.0:1787829734.269014:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.269017:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829734.269020:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829734.269022:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829734.269024:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829734.269025:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.269026:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829734.269027:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829734.269028:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829734.269031:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829734.269034:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829734.269038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787829734.269041:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000100:00000001:3.0:1787829734.269048:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787829734.269050:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000001:3.0:1787829734.269051:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829734.269057:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829734.269060:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373696, portal 25
00000100:00000001:2.0:1787829734.269066:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787829734.269068:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269069:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00080000:3.0:1787829734.269072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829734.269075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829734.269077:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:2.0:1787829734.269078:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829734.269079:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829734.269085:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787829734.269092:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829734.269098:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829734.269098:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3881
00000400:00000010:2.0:1787829734.269104:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000400:00000200:3.0:1787829734.269105:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829734.269110:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000200:2.0:1787829734.269112:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327373696, offset 0
00000100:00080000:3.0:1787829734.269113:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829734.269118:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269121:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829734.269124:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269127:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269127:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829734.269133:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829734.269134:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829734.269136:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829734.269138:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:2.0:1787829734.269139:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:3.0:1787829734.269144:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:2.0:1787829734.269149:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000001:3.0:1787829734.269150:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:2.0:1787829734.269154:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1787829734.269158:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269161:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:3.0:1787829734.269164:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787829734.269167:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1787829734.269171:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269172:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269177:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:2.0:1787829734.269178:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40691:40691:256:4294967295] 192.168.203.116@tcp LPNI seq info [40691:40691:8:4294967295]
00000020:00000040:3.0:1787829734.269184:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:2.0:1787829734.269186:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829734.269190:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269193:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829734.269197:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787829734.269201:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1787829734.269202:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829734.269206:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787829734.269211:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269211:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829734.269216:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829734.269218:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1787829734.269219:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000100:00000001:3.0:1787829734.269220:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787829734.269226:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:3.0:1787829734.269226:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000800:00000200:2.0:1787829734.269231:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000020:00000040:3.0:1787829734.269231:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829734.269237:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829734.269240:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787829734.269241:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:3.0:1787829734.269243:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1787829734.269244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
02000000:00000001:3.0:1787829734.269246:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829734.269249:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1787829734.269253:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000010:3.0:1787829734.269256:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000800:00000010:2.0:1787829734.269257:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000100:00000001:3.0:1787829734.269262:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829734.269264:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:3.0:1787829734.269270:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787829734.269271:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829734.269278:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000100:00000001:3.0:1787829734.269280:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787829734.269282:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:3.0:1787829734.269283:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269297:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787829734.269303:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829734.269306:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829734.269323:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269327:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1787829734.269327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829734.269332:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000040:2.0:1787829734.269333:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829734.269335:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829734.269338:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829734.269343:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787829734.269345:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829734.269348:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3881
00000100:00000001:2.0:1787829734.269349:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:3.0:1787829734.269354:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829734.269357:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829734.269359:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829734.269361:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269363:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829734.269364:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829734.269365:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.269366:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829734.269371:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829734.269373:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000200:2.0:1787829734.269374:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:3.0:1787829734.269378:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787829734.269380:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298555052
00000020:00000040:3.0:1787829734.269381:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829734.269384:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000010:2.0:1787829734.269385:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee95a0.
00000020:00000001:3.0:1787829734.269386:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829734.269392:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269393:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787829734.269395:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787829734.269395:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829734.269400:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:2.0:1787829734.269402:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298555052
00000020:00000040:3.0:1787829734.269402:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829734.269405:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000010:2.0:1787829734.269408:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87ee92a0.
00000100:00000001:3.0:1787829734.269409:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829734.269413:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787829734.269414:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:3.0:1787829734.269416:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829734.269418:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829734.269419:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269422:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829734.269423:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829734.269426:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787829734.269427:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000008:00000001:3.0:1787829734.269430:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829734.269434:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829734/real 1787829734]  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829734.269434:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829734.269438:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829734.269443:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000001:2.0:1787829734.269447:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:3.0:1787829734.269449:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:2.0:1787829734.269450:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:3.0:1787829734.269453:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787829734.269454:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829734.269458:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000100:00000001:3.0:1787829734.269459:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787829734.269460:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829734.269464:0:2346:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1787829734.269464:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829734.269469:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787829734.269470:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787829734.269478:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000200:2.0:1787829734.269479:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269486:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787829734.269486:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269489:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269492:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.269493:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787829734.269497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1787829734.269498:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829734.269518:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829734.269520:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829734.269523:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829734.269527:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829734.269531:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269542:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829734.269548:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829734.269549:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269550:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.269553:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829734.269554:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3892)
00000100:00000001:2.0:1787829734.269555:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829734.269560:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829734.269562:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829734.269565:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829734.269567:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829734.269570:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829734.269576:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327373696:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787829734.269580:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269582:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.269583:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327373696/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269587:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829734.269589:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829734.269591:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829734.269593:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829734.269594:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829734.269595:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:2.0:1787829734.269597:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829734.269598:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:2.0:1787829734.269601:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829734.269602:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829734.269603:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:2.0:1787829734.269605:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829734.269606:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269607:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269613:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829734.269614:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269616:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829734.269617:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829734.269618:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829734.269620:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269629:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829734.269630:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829734.269631:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829734.269633:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829734.269634:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829734.269636:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373824:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829734.269638:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829734.269639:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829734.269640:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829734.269641:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829734.269643:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829734.269644:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829734.269647:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829734.269649:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829734.269651:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829734.269653:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373824, portal 10
00000100:00000001:2.0:1787829734.269671:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829734.269672:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000100:00000040:2.0:1787829734.269678:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269682:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829734.269684:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829734.269686:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829734.269688:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327373824, offset 0
00000400:00000200:2.0:1787829734.269690:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269693:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829734.269696:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829734.269698:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829734.269700:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829734.269701:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829734.269703:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269706:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40692:40692:256:4294967295] 192.168.203.116@tcp LPNI seq info [40692:40692:8:4294967295]
00000400:00000200:2.0:1787829734.269708:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829734.269713:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829734.269716:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829734.269718:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829734.269721:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829734.269723:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787829734.269726:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829734.269728:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787829734.269730:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829734.269731:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829734.269733:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829734.269736:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829734.269738:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829734.269739:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829734.269741:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829734.269743:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269749:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.269750:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269755:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269756:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829734.269758:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829734.269760:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:2.0:1787829734.269762:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829734.269764:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298554379
00000100:00000001:2.0:1787829734.269766:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298555052
00000100:00000001:2.0:1787829734.269767:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298554379
00000100:00000001:2.0:1787829734.269769:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298555052
00000100:00000001:2.0:1787829734.269770:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829734.269772:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269773:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829734.269775:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829734/real 1787829734]  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829734.269780:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829734.269781:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829734.269782:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829734.269784:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829734.269785:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829734.269786:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829734.269788:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829734.269793:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269797:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829734.269798:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269799:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.269800:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829734.269802:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269806:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829734.269808:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269812:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829734.269814:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269815:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.269816:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269817:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829734.269818:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829734.269825:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829734.269827:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829734.269829:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829734.269832:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829734.269837:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373824:192.168.203.116@tcp:38:kworker.0
00000100:00000001:2.0:1787829734.269839:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829734.269841:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269845:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829734.269846:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829734.269847:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829734.269848:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829734.269850:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829734.269851:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373952:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829734.269854:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829734.269855:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829734.269856:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829734.269857:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829734.269859:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829734.269860:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829734.269862:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829734.269864:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829734.269866:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829734.269868:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327373952, portal 4
00000100:00000001:2.0:1787829734.269869:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829734.269870:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:2.0:1787829734.269874:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269878:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829734.269880:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829734.269881:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829734.269883:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327373952, offset 0
00000400:00000200:2.0:1787829734.269885:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269887:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829734.269889:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829734.269891:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829734.269893:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829734.269894:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829734.269896:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829734.269898:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40693:40693:256:4294967295] 192.168.203.116@tcp LPNI seq info [40693:40693:8:4294967295]
00000400:00000200:2.0:1787829734.269901:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829734.269905:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829734.269907:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829734.269909:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829734.269911:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829734.269913:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787829734.269916:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829734.269917:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787829734.269920:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829734.269921:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829734.269923:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829734.269925:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829734.269927:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829734.269928:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829734.269929:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829734.269932:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269936:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.269938:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269942:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269943:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829734.269946:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829734.269947:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:2.0:1787829734.269949:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829734.269951:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298554386
00000100:00000001:2.0:1787829734.269952:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298555052
00000100:00000001:2.0:1787829734.269954:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298554386
00000100:00000001:2.0:1787829734.269956:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298555052
00000100:00000001:2.0:1787829734.269957:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829734.269958:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.269959:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829734.269961:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829734/real 1787829734]  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829734.269968:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829734.269969:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829734.269971:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829734.269973:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829734.269974:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829734.269976:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829734.269980:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829734.269988:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.269996:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829734.269997:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.269999:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.270001:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829734.270005:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270013:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829734.270017:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270023:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829734.270025:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270026:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.270027:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270028:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829734.270029:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829734.270031:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829734.270033:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829734.270034:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829734.270037:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829734.270042:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327373952:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829734.270044:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829734.270046:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270050:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829734.270051:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829734.270052:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829734.270054:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829734.270055:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829734.270057:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374080:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829734.270059:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829734.270060:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829734.270061:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829734.270062:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787829734.270064:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787829734.270066:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829734.270068:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:2.0:1787829734.270070:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dda03f7210.
00000400:00000010:2.0:1787829734.270072:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6198.
00000100:00000200:2.0:1787829734.270074:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374080, portal 4
00000100:00000001:2.0:1787829734.270075:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829734.270076:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:2.0:1787829734.270080:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270086:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829734.270088:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:2.0:1787829734.270090:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd85ce6110.
00000100:00000200:2.0:1787829734.270093:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327374080, offset 0
00000400:00000200:2.0:1787829734.270097:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829734.270100:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829734.270104:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829734.270107:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829734.270109:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829734.270112:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787829734.270114:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829734.270118:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40694:40694:256:4294967295] 192.168.203.116@tcp LPNI seq info [40694:40694:8:4294967295]
00000400:00000200:2.0:1787829734.270122:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829734.270129:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787829734.270134:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829734.270136:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99b37a00.
00000800:00000200:2.0:1787829734.270139:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.0:1787829734.270141:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787829734.270144:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:2.1:1787829734.270145:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787829734.270148:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:2.0:1787829734.270149:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99b37a00.
00000400:00000200:2.0:1787829734.270151:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787829734.270154:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787829734.270156:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6110
00000400:00000010:2.0:1787829734.270157:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6110.
00000100:00000001:2.0:1787829734.270159:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829734.270161:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270166:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.270168:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270173:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.270174:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829734.270176:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829734.270178:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:2.0:1787829734.270180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829734.270181:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298554391
00000100:00000001:2.0:1787829734.270183:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298555052
00000100:00000001:2.0:1787829734.270184:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298554391
00000100:00000001:2.0:1787829734.270186:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298555052
00000100:00000001:2.0:1787829734.270187:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829734.270188:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.270190:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829734.270192:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829734/real 1787829734]  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787829734.270196:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787829734.270197:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787829734.270199:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dda03f7210.
00000400:00000200:2.0:1787829734.270200:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd85ce6198
00000400:00000010:2.0:1787829734.270201:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd85ce6198.
00000100:00000001:2.0:1787829734.270202:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787829734.270204:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829734.270210:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270218:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829734.270219:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270220:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.270221:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829734.270223:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270227:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829734.270230:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270234:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829734.270235:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270236:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829734.270237:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270238:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829734.270239:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787829734.270241:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787829734.270243:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829734.270245:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829734.270247:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829734.270251:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374080:192.168.203.116@tcp:8:kworker.0
00000100:00000001:2.0:1787829734.270253:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270255:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.270256:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327373824/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270260:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829734.270261:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829734.270263:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87000400.
02000000:00000001:2.0:1787829734.270265:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829734.270266:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829734.270267:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:2.0:1787829734.270268:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829734.270270:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:2.0:1787829734.270271:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829734.270272:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829734.270273:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:2.0:1787829734.270275:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829734.270276:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270277:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.270279:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327373952/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270283:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829734.270284:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829734.270285:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87002800.
02000000:00000001:2.0:1787829734.270287:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829734.270307:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829734.270308:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:2.0:1787829734.270310:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829734.270311:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:2.0:1787829734.270313:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829734.270313:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829734.270314:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:2.0:1787829734.270316:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829734.270317:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270318:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829734.270321:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327374080/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829734.270325:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787829734.270326:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787829734.270327:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd87001400.
02000000:00000001:2.0:1787829734.270328:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787829734.270329:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787829734.270330:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:2.0:1787829734.270331:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787829734.270332:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:2.0:1787829734.270334:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787829734.270334:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787829734.270335:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:2.0:1787829734.270338:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787829734.270338:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270339:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829734.270342:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829734.270343:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829734.270344:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829734.270345:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829734.780750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829734.780756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829734.780758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829734.780763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829734.780768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829734.780772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829734.780774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829734.780777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829734.780780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829734.972755:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298555052, at 4298555072
00000100:00000040:2.0:1787829734.972758:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298555052, at 4298555072
00000100:00000010:3.0:1787829734.972769:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee92a0.
00000100:00000010:2.0:1787829734.972771:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87ee95a0.
00010000:00000001:0.0:1787829735.100755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829735.100762:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829735.100764:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829735.100767:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100771:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829735.100773:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100775:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100776:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100780:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829735.100781:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829735.100782:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829735.100784:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100785:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829735.100786:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100787:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100789:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829735.100792:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829735.100793:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829735.100828:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829735.100833:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829735.100837:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829735.804777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829735.804782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829735.804784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829735.804788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829735.804791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829735.804794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829735.804796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829735.804798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829735.804800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829736.124788:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829736.124794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829736.124799:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829736.124801:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829736.124804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829736.124805:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829736.124826:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829736.124828:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829736.124830:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829736.828744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829736.828750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829736.828752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829736.828755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829736.828759:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829736.828762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829736.828764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829736.828766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829736.828768:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829737.148793:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829737.148799:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829737.148803:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829737.148805:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829737.148808:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829737.148810:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829737.148864:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829737.148869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829737.148873:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829737.852754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829737.852761:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829737.852764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829737.852769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829737.852773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829737.852777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829737.852780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829737.852783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829737.852786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829738.172741:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829738.172746:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829738.172751:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829738.172753:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829738.172756:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829738.172757:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829738.172796:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829738.172799:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829738.172802:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829738.876741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829738.876748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829738.876751:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829738.876756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829738.876761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829738.876764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829738.876767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829738.876770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829738.876773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829739.196768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829739.196774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829739.196780:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829739.196782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829739.196786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829739.196788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829739.196814:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829739.196817:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829739.196821:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829739.388828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829739.388836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829739.388838:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829739.388841:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829739.388844:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829739.388851:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3887
00000400:00000200:2.0:1787829739.388867:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829739.388889:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829739.388892:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829739.388895:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.388897:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.388899:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.388900:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829739.388903:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829739.388905:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829739.388907:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829739.388910:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829739.388912:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829739.388914:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829739.388916:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.388917:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.388918:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.388920:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829739.388922:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829739.388924:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829739.388925:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829739.388928:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829739.388930:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829739.388932:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.388934:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829739.388936:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829739.388944:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829739.388946:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829739.388950:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829739.388951:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.388953:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829739.388955:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829739.388959:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829739.388965:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829739.388971:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829739.388979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829739.388987:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829739.388990:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829739.388993:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829739.389013:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829739.389015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829739.389018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829739.389019:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829739.389021:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829739.389023:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829739.389025:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3887
00000400:00000200:2.0:1787829739.389027:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829739.389029:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829739.389031:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829739.389032:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389034:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.389035:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389037:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829739.389038:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829739.389039:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829739.389040:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829739.389041:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829739.389043:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829739.389044:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829739.389046:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389047:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.389048:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389050:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829739.389051:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829739.389053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389054:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829739.389055:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829739.389059:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829739.389060:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389062:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829739.389065:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829739.389068:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389070:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829739.389072:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787829739.389074:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829739.389076:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829739.389077:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389079:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829739.389080:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829739.389082:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829739.389084:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829739.389086:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389106:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00100000:2.0:1787829739.389108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829739.389113:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829739.389118:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787829739.389121:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829739.389125:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829739.389130:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829739.389133:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389156:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829739.389159:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389160:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829739.389164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829739.389166:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:2.0:1787829739.389171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
02000000:00000001:0.0:1787829739.389172:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829739.389173:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829739.389176:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:0.0:1787829739.389178:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389180:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:0.0:1787829739.389187:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327374208:192.168.203.116@tcp:250:kworker.0
00000100:00080000:2.0:1787829739.389187:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3887
00000400:00000200:2.0:1787829739.389192:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829739.389197:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829739.389198:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787829739.389200:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829739.389203:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829739.389204:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787829739.389205:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.389209:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829739.389210:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389212:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829739.389215:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:2.0:1787829739.389216:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829739.389219:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829739.389221:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829739.389222:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829739.389224:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
02000000:00000010:0.0:1787829739.389225:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
00000020:00000001:2.0:1787829739.389226:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
02000000:00000001:0.0:1787829739.389231:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829739.389231:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389236:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.389238:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389239:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829739.389243:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829739.389244:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829739.389247:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389250:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829739.389267:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
02000000:00000001:2.0:1787829739.389267:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829739.389272:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829739.389275:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389276:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:0.0:1787829739.389278:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000008:00100000:2.0:1787829739.389278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829739.389284:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389285:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829739.389289:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000100:00000200:0.0:1787829739.389290:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374208, portal 25
00000020:00000040:2.0:1787829739.389291:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:0.0:1787829739.389295:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1787829739.389296:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829739.389297:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389300:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787829739.389300:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829739.389302:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829739.389305:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829739.389309:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000040:0.0:1787829739.389312:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829739.389313:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829739.389321:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829739.389329:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829739.389338:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389343:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1787829739.389344:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000001:2.0:1787829739.389345:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829739.389353:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327374208, offset 0
00000100:00000040:2.0:1787829739.389353:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829739.389363:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829739.389367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829739.389368:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829739.389372:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829739.389374:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829739.389376:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829739.389378:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829739.389382:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3887
00000400:00000200:2.0:1787829739.389386:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829739.389389:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829739.389394:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829739.389397:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829739.389400:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389405:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829739.389408:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:2.0:1787829739.389412:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829739.389417:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787829739.389418:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829739.389423:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829739.389425:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000040:2.0:1787829739.389426:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829739.389430:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829739.389432:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:2.0:1787829739.389434:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829739.389438:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829739.389439:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000001:2.0:1787829739.389441:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829739.389447:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40695:40695:256:4294967295] 192.168.203.116@tcp LPNI seq info [40695:40695:8:4294967295]
00000100:00000040:2.0:1787829739.389448:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389453:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829739.389457:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829739.389461:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829739.389462:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829739.389467:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829739.389471:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389476:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829739.389481:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829739.389482:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1787829739.389486:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.0:1787829739.389492:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829739.389493:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389498:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:0.0:1787829739.389501:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000008:00100000:2.0:1787829739.389501:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829739.389507:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389510:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829739.389512:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:2.0:1787829739.389517:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000800:00000200:0.0:1787829739.389518:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000020:00000040:2.0:1787829739.389523:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829739.389529:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.1:1787829739.389530:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829739.389531:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829739.389535:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829739.389538:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.1:1787829739.389538:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
02000000:00000001:2.0:1787829739.389541:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829739.389546:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000800:00000100:0.0:1787829739.389548:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829739.389550:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829739.389552:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000100:00100000:2.0:1787829739.389557:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829739.389561:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787829739.389565:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829739.389569:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829739.389572:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1787829739.389576:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829739.389580:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829739.389585:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:2.0:1787829739.389586:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389593:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787829739.389593:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3897)
00000100:00000200:0.0:1787829739.389602:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389614:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.389619:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389627:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389628:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829739.389634:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829739.389638:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829739.389640:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298560172
00000100:00000010:0.0:1787829739.389645:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc300.
00000100:00000001:0.0:1787829739.389652:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829739.389689:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298560173
00000100:00000010:0.0:1787829739.389694:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc7e0.
00000100:00000001:0.0:1787829739.389697:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829739.389701:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829739.389704:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389706:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829739.389710:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739]  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829739.389720:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829739.389721:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829739.389723:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829739.389726:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829739.389728:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829739.389731:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829739.389734:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829739.389744:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389751:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829739.389753:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.389755:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.389758:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829739.389762:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389769:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829739.389774:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389783:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829739.389786:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.389787:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.389789:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.389791:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829739.389794:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829739.389796:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829739.389801:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829739.389804:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829739.389809:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829739.389817:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327374208:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829739.389821:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.389822:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.389824:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327374208/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389829:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829739.389831:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829739.389833:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829739.389835:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829739.389837:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829739.389838:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829739.389840:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829739.389842:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829739.389845:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829739.389847:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829739.389848:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:0.0:1787829739.389850:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829739.389851:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.389853:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.389859:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829739.389860:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389862:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829739.389863:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829739.389864:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829739.389866:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389871:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829739.389872:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829739.389874:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829739.389875:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829739.389876:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829739.389878:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374336:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829739.389880:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829739.389882:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829739.389883:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829739.389884:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829739.389886:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829739.389888:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829739.389890:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829739.389892:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829739.389895:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829739.389897:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374336, portal 10
00000100:00000001:0.0:1787829739.389898:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829739.389899:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:0.0:1787829739.389904:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389909:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829739.389910:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829739.389912:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829739.389913:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327374336, offset 0
00000400:00000200:0.0:1787829739.389916:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829739.389919:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829739.389921:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829739.389923:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829739.389925:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829739.389927:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829739.389928:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829739.389930:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40696:40696:256:4294967295] 192.168.203.116@tcp LPNI seq info [40696:40696:8:4294967295]
00000400:00000200:0.0:1787829739.389933:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829739.389938:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829739.389941:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829739.389943:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829739.389946:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829739.389948:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829739.389951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829739.389953:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829739.389955:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829739.389957:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829739.389959:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829739.389961:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829739.389963:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829739.389964:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829739.389966:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829739.389968:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389973:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.389975:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.389984:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.389985:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829739.389987:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829739.389990:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829739.389992:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829739.389993:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298559499
00000100:00000001:0.0:1787829739.389995:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298560172
00000100:00000001:0.0:1787829739.389997:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298559499
00000100:00000001:0.0:1787829739.389998:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298560173
00000100:00000001:0.0:1787829739.389999:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829739.390000:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.390001:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829739.390004:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739]  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829739.390008:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829739.390009:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829739.390010:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829739.390012:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829739.390013:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829739.390014:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829739.390016:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829739.390021:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390025:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829739.390026:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390027:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.390028:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829739.390030:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390034:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829739.390037:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390041:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829739.390043:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390044:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.390045:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390046:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829739.390047:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829739.390048:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829739.390050:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829739.390052:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829739.390054:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829739.390059:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374336:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829739.390061:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829739.390063:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390068:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829739.390069:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829739.390070:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829739.390071:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829739.390072:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829739.390074:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374464:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829739.390076:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829739.390077:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829739.390078:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829739.390080:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829739.390083:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829739.390085:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829739.390089:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829739.390091:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829739.390095:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829739.390097:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374464, portal 4
00000100:00000001:0.0:1787829739.390099:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829739.390101:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:0.0:1787829739.390106:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390113:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829739.390116:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829739.390119:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829739.390121:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327374464, offset 0
00000400:00000200:0.0:1787829739.390125:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829739.390130:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829739.390133:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829739.390136:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829739.390139:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829739.390141:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829739.390144:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829739.390147:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40697:40697:256:4294967295] 192.168.203.116@tcp LPNI seq info [40697:40697:8:4294967295]
00000400:00000200:0.0:1787829739.390151:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829739.390158:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829739.390163:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829739.390166:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829739.390170:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829739.390173:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829739.390178:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829739.390181:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829739.390185:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829739.390188:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829739.390191:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829739.390194:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829739.390198:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829739.390199:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829739.390202:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829739.390206:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390215:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.390218:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390225:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.390227:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829739.390230:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829739.390233:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829739.390236:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829739.390238:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298559506
00000100:00000001:0.0:1787829739.390241:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298560172
00000100:00000001:0.0:1787829739.390243:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298559506
00000100:00000001:0.0:1787829739.390245:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298560173
00000100:00000001:0.0:1787829739.390248:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829739.390250:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.390265:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829739.390270:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739]  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829739.390279:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829739.390280:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829739.390283:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829739.390285:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829739.390287:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829739.390290:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829739.390293:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829739.390303:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390310:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829739.390311:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390313:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.390315:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829739.390319:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390327:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829739.390331:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390338:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829739.390341:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390343:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.390344:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390346:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829739.390347:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829739.390350:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829739.390354:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829739.390357:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829739.390360:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829739.390369:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374464:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829739.390373:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829739.390376:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390383:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829739.390385:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829739.390387:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829739.390389:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829739.390391:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829739.390394:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374592:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829739.390397:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829739.390399:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829739.390401:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829739.390403:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829739.390406:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829739.390408:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829739.390411:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829739.390414:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829739.390418:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829739.390421:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374592, portal 4
00000100:00000001:0.0:1787829739.390423:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829739.390424:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:0.0:1787829739.390430:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390436:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829739.390438:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829739.390441:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829739.390444:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327374592, offset 0
00000400:00000200:0.0:1787829739.390448:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829739.390452:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829739.390456:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829739.390459:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829739.390462:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829739.390464:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829739.390466:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829739.390469:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40698:40698:256:4294967295] 192.168.203.116@tcp LPNI seq info [40698:40698:8:4294967295]
00000400:00000200:0.0:1787829739.390488:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829739.390495:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829739.390500:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829739.390504:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829739.390508:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829739.390510:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829739.390515:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829739.390518:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829739.390522:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829739.390524:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829739.390527:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829739.390531:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829739.390534:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829739.390536:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829739.390538:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829739.390542:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390551:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.390554:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390562:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.390564:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829739.390567:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829739.390570:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829739.390573:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829739.390576:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298559511
00000100:00000001:0.0:1787829739.390578:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298560172
00000100:00000001:0.0:1787829739.390581:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298559511
00000100:00000001:0.0:1787829739.390583:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298560173
00000100:00000001:0.0:1787829739.390585:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829739.390588:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.390590:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829739.390594:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739]  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829739.390602:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829739.390604:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829739.390606:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829739.390608:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829739.390610:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829739.390612:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829739.390615:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829739.390625:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390632:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829739.390634:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390636:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.390637:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829739.390641:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390648:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829739.390652:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390679:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829739.390682:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390684:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829739.390686:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390688:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829739.390689:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829739.390692:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829739.390697:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829739.390699:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829739.390703:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829739.390713:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374592:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829739.390716:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390719:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.390722:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327374336/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390730:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829739.390731:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829739.390733:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829739.390737:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829739.390738:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829739.390740:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829739.390742:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829739.390744:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829739.390747:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829739.390748:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829739.390750:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:0.0:1787829739.390753:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829739.390755:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390756:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.390760:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327374464/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390767:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829739.390768:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829739.390770:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829739.390772:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829739.390774:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829739.390775:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829739.390777:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829739.390779:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829739.390782:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829739.390783:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829739.390784:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:0.0:1787829739.390787:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829739.390788:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390790:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829739.390794:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327374592/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829739.390801:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829739.390802:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829739.390804:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829739.390806:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829739.390808:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829739.390809:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829739.390811:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829739.390813:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829739.390815:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829739.390816:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829739.390818:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:0.0:1787829739.390820:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829739.390821:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390823:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.390827:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829739.390828:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.390831:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829739.390832:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829739.900770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829739.900781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829739.900784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829739.900792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829739.900796:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829739.900801:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.900804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829739.900807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829739.900810:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829740.092799:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298560172, at 4298560192
00000100:00000040:3.0:1787829740.092799:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298560173, at 4298560192
00000100:00000010:2.0:1787829740.092818:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000100:00000010:3.0:1787829740.092820:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00010000:00000001:0.0:1787829740.220753:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829740.220763:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829740.220771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829740.220772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829740.220777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829740.220781:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829740.220784:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829740.220788:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829740.220792:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829740.220795:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829740.220798:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829740.220801:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829740.220885:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829740.220890:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829740.220894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829740.924810:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829740.924818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829740.924820:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829740.924825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829740.924828:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829740.924832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829740.924835:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829740.924837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829740.924840:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829741.244774:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829741.244782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829741.244789:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829741.244790:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829741.244795:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829741.244796:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829741.244837:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829741.244841:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829741.244844:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829741.948825:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829741.948833:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829741.948835:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829741.948840:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829741.948843:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829741.948849:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829741.948851:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829741.948854:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829741.948856:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829742.268748:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829742.268771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829742.268779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829742.268782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829742.268786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829742.268788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829742.268816:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829742.268818:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829742.268821:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829742.972839:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829742.972847:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829742.972849:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829742.972855:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829742.972858:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829742.972862:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829742.972865:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829742.972868:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829742.972870:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829743.292813:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829743.292820:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829743.292825:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829743.292826:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829743.292830:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829743.292832:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829743.292883:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829743.292887:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829743.292891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829743.996804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829743.996812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829743.996814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829743.996819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829743.996822:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829743.996826:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829743.996828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829743.996831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829743.996833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316751:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829744.316759:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829744.316762:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829744.316764:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316769:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829744.316771:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316773:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316774:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316778:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829744.316780:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829744.316781:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829744.316782:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316783:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829744.316784:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316785:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316787:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829744.316789:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829744.316790:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829744.316832:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829744.316836:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829744.316839:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829744.508785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829744.508800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829744.508803:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829744.508808:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829744.508814:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829744.508824:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3892
00000400:00000200:3.0:1787829744.508835:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829744.508841:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829744.508845:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829744.508848:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.508850:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829744.508853:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.508855:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829744.508859:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829744.508862:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829744.508864:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829744.508867:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829744.508869:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829744.508871:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829744.508873:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.508874:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829744.508876:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.508877:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829744.508880:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829744.508883:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829744.508884:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829744.508887:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829744.508890:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829744.508892:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.508894:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829744.508896:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829744.508907:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829744.508910:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829744.508913:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829744.508915:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.508918:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829744.508921:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829744.508926:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829744.508934:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829744.508940:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829744.508949:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829744.508958:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829744.508963:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829744.508967:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829744.508987:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829744.508990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829744.508992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829744.508994:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829744.508995:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829744.508997:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829744.508999:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3892
00000400:00000200:3.0:1787829744.509001:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829744.509003:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829744.509004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829744.509006:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509008:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829744.509010:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509012:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829744.509013:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829744.509014:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829744.509016:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829744.509017:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829744.509019:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829744.509020:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829744.509022:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509024:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829744.509026:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509027:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829744.509029:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829744.509031:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829744.509034:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787829744.509035:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829744.509039:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829744.509043:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:3.0:1787829744.509045:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829744.509048:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1787829744.509048:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509053:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829744.509057:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829744.509060:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829744.509063:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829744.509063:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829744.509068:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829744.509072:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829744.509077:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829744.509079:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509085:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829744.509086:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787829744.509087:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829744.509091:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829744.509092:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.509101:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:3.0:1787829744.509102:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829744.509107:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787829744.509107:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.509114:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829744.509115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829744.509122:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327374720:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829744.509128:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829744.509132:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509133:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1787829744.509137:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829744.509141:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829744.509146:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509147:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829744.509153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829744.509154:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1787829744.509156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829744.509159:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829744.509160:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829744.509161:0:36:0:(import.c:485:import_select_connection()) Process entered
02000000:00000010:0.0:1787829744.509163:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531000.
00000100:00080000:3.0:1787829744.509163:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3892
02000000:00000001:0.0:1787829744.509170:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787829744.509170:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829744.509174:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829744.509177:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829744.509178:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509180:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829744.509184:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829744.509186:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829744.509191:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000100:00000001:3.0:1787829744.509191:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829744.509194:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829744.509198:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829744.509200:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1787829744.509203:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000020:00000040:3.0:1787829744.509203:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829744.509210:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829744.509213:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829744.509229:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829744.509230:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374720, portal 25
00000100:00000001:3.0:1787829744.509234:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829744.509236:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1787829744.509240:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829744.509241:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000001:3.0:1787829744.509246:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829744.509252:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000040:0.0:1787829744.509254:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829744.509259:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509267:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829744.509272:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829744.509277:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829744.509283:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509285:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787829744.509286:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829744.509291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000040:0.0:1787829744.509292:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000008:00000001:3.0:1787829744.509296:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829744.509299:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000001:3.0:1787829744.509300:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829744.509307:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327374720, offset 0
00000100:00000010:3.0:1787829744.509308:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829744.509313:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829744.509316:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829744.509320:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829744.509321:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509326:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829744.509329:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829744.509331:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509335:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:3.0:1787829744.509337:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829744.509343:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509345:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00100000:3.0:1787829744.509352:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829744.509354:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787829744.509364:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829744.509368:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509372:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000040:3.0:1787829744.509374:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829744.509381:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1787829744.509385:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509390:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:3.0:1787829744.509391:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829744.509396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829744.509399:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829744.509401:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40699:40699:256:4294967295] 192.168.203.116@tcp LPNI seq info [40699:40699:8:4294967295]
00000100:00080000:3.0:1787829744.509402:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829744.509406:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829744.509411:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3892
00000400:00000200:0.0:1787829744.509416:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829744.509418:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829744.509422:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829744.509424:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829744.509426:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509428:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829744.509433:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509435:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829744.509436:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829744.509437:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829744.509438:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829744.509440:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829744.509444:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829744.509446:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000001:3.0:1787829744.509448:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829744.509454:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509455:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1787829744.509460:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:3.0:1787829744.509460:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509462:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829744.509466:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000010:0.0:1787829744.509468:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00080000:3.0:1787829744.509469:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829744.509474:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829744.509476:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829744.509478:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.0:1787829744.509482:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:3.0:1787829744.509483:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509487:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829744.509488:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000008:00100000:3.0:1787829744.509489:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829744.509494:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509497:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829744.509498:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:3.0:1787829744.509502:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000800:00000200:0.1:1787829744.509503:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000020:00000040:3.0:1787829744.509506:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000800:00000100:0.0:1787829744.509511:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829744.509513:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:0.0:1787829744.509515:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
02000000:00000001:3.0:1787829744.509516:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829744.509519:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829744.509520:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829744.509521:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509522:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:3.0:1787829744.509525:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000400:00000200:0.0:1787829744.509530:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787829744.509532:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509536:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000100:00100000:3.0:1787829744.509539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829744.509540:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829744.509546:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787829744.509550:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:0.0:1787829744.509554:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829744.509555:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829744.509564:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509571:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787829744.509574:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829744.509578:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829744.509580:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3902)
00000100:00000001:0.0:1787829744.509593:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509596:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829744.509602:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509606:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829744.509609:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298565292
00000100:00000010:0.0:1787829744.509613:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc4e0.
00000100:00000001:0.0:1787829744.509619:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829744.509624:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298565292
00000100:00000010:0.0:1787829744.509626:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc0c0.
00000100:00000001:0.0:1787829744.509629:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829744.509636:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829744.509639:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509641:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829744.509644:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744]  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829744.509652:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829744.509653:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829744.509684:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829744.509687:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829744.509688:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829744.509690:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829744.509693:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829744.509699:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509705:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829744.509706:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509708:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.509709:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829744.509712:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509718:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829744.509721:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509726:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829744.509728:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509729:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.509730:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509732:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829744.509734:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829744.509736:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829744.509743:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829744.509745:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829744.509748:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829744.509754:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327374720:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829744.509758:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509760:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.509762:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327374720/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509766:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829744.509769:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829744.509771:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829744.509773:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829744.509774:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829744.509775:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829744.509777:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829744.509779:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:0.0:1787829744.509783:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829744.509786:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829744.509787:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829744.509790:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829744.509791:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509792:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509799:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829744.509800:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509802:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829744.509803:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829744.509804:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829744.509807:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509812:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829744.509813:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.509814:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829744.509816:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829744.509817:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829744.509819:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374848:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829744.509821:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829744.509823:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829744.509824:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.509825:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829744.509828:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829744.509829:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.509832:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829744.509834:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829744.509836:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829744.509839:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374848, portal 10
00000100:00000001:0.0:1787829744.509840:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829744.509841:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:0.0:1787829744.509847:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509852:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829744.509854:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829744.509856:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829744.509857:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327374848, offset 0
00000400:00000200:0.0:1787829744.509860:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829744.509863:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829744.509866:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829744.509868:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829744.509870:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829744.509872:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829744.509874:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829744.509876:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40700:40700:256:4294967295] 192.168.203.116@tcp LPNI seq info [40700:40700:8:4294967295]
00000400:00000200:0.0:1787829744.509879:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829744.509884:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829744.509887:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829744.509889:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829744.509892:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829744.509894:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829744.509897:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829744.509898:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829744.509901:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829744.509902:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829744.509904:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829744.509907:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829744.509909:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829744.509910:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829744.509912:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829744.509914:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509920:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.509922:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509927:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509928:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829744.509930:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829744.509933:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829744.509935:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829744.509937:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298564619
00000100:00000001:0.0:1787829744.509939:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298565292
00000100:00000001:0.0:1787829744.509941:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298564619
00000100:00000001:0.0:1787829744.509943:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298565292
00000100:00000001:0.0:1787829744.509944:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829744.509945:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.509946:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829744.509948:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744]  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829744.509953:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829744.509954:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829744.509956:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829744.509957:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829744.509958:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829744.509960:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829744.509963:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829744.509972:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509980:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829744.509981:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.509983:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.509984:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829744.509989:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.509996:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829744.510001:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510009:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829744.510012:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510014:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.510015:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510017:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829744.510019:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829744.510022:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829744.510025:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829744.510028:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829744.510033:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829744.510042:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374848:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829744.510045:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829744.510047:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510052:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829744.510053:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.510055:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829744.510056:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829744.510057:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829744.510059:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374976:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829744.510061:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829744.510063:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829744.510065:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.510067:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829744.510071:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829744.510073:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.510077:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829744.510080:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829744.510084:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829744.510087:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327374976, portal 4
00000100:00000001:0.0:1787829744.510089:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829744.510090:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829744.510097:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510104:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829744.510107:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829744.510110:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829744.510113:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327374976, offset 0
00000400:00000200:0.0:1787829744.510117:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829744.510121:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829744.510125:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829744.510128:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829744.510131:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829744.510134:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829744.510136:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829744.510139:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40701:40701:256:4294967295] 192.168.203.116@tcp LPNI seq info [40701:40701:8:4294967295]
00000400:00000200:0.0:1787829744.510142:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829744.510147:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829744.510150:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829744.510153:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829744.510156:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829744.510158:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829744.510161:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829744.510162:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829744.510166:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829744.510167:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829744.510170:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829744.510172:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829744.510174:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829744.510175:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829744.510178:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829744.510181:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510186:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.510189:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510193:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.510194:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829744.510197:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829744.510199:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829744.510201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829744.510203:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298564626
00000100:00000001:0.0:1787829744.510205:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298565292
00000100:00000001:0.0:1787829744.510206:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298564626
00000100:00000001:0.0:1787829744.510208:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298565292
00000100:00000001:0.0:1787829744.510210:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829744.510211:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.510212:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829744.510214:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744]  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829744.510238:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829744.510239:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829744.510240:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829744.510242:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829744.510243:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829744.510245:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829744.510248:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829744.510253:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510262:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829744.510263:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510265:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.510266:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829744.510268:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510272:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829744.510275:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510280:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829744.510282:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510283:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.510283:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510285:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829744.510285:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829744.510287:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829744.510289:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829744.510290:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829744.510293:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829744.510297:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327374976:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829744.510300:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829744.510302:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510307:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829744.510308:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.510310:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829744.510311:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829744.510313:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829744.510315:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375104:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829744.510317:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829744.510319:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829744.510320:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829744.510321:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829744.510323:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829744.510325:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829744.510327:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829744.510329:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829744.510332:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829744.510334:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375104, portal 4
00000100:00000001:0.0:1787829744.510335:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829744.510336:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:0.0:1787829744.510340:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510344:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829744.510346:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829744.510347:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829744.510349:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327375104, offset 0
00000400:00000200:0.0:1787829744.510352:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829744.510354:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829744.510357:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829744.510359:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829744.510361:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829744.510363:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829744.510364:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829744.510367:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40702:40702:256:4294967295] 192.168.203.116@tcp LPNI seq info [40702:40702:8:4294967295]
00000400:00000200:0.0:1787829744.510370:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829744.510375:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829744.510378:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829744.510381:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829744.510383:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829744.510385:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829744.510388:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829744.510405:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829744.510408:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829744.510409:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829744.510412:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829744.510414:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829744.510416:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829744.510418:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829744.510420:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829744.510422:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510428:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.510430:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510434:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.510435:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829744.510437:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829744.510440:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829744.510442:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829744.510444:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298564631
00000100:00000001:0.0:1787829744.510445:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298565292
00000100:00000001:0.0:1787829744.510447:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298564631
00000100:00000001:0.0:1787829744.510448:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298565292
00000100:00000001:0.0:1787829744.510450:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829744.510451:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.510452:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829744.510455:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744]  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829744.510461:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829744.510462:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829744.510463:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829744.510465:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829744.510466:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829744.510467:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829744.510469:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829744.510474:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510479:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829744.510480:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510481:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.510482:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829744.510484:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510488:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829744.510490:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510494:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829744.510495:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510496:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829744.510497:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510498:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829744.510499:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829744.510501:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829744.510503:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829744.510504:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829744.510507:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829744.510513:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375104:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829744.510515:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510516:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.510518:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327374848/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510522:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829744.510523:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829744.510525:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829744.510526:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829744.510527:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829744.510528:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829744.510529:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829744.510530:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829744.510532:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829744.510532:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829744.510533:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829744.510535:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829744.510536:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510537:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.510539:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327374976/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510543:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829744.510544:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829744.510545:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829744.510547:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829744.510548:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829744.510548:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829744.510549:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829744.510550:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829744.510551:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829744.510553:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829744.510554:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829744.510555:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829744.510556:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510558:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829744.510560:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327375104/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829744.510563:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829744.510564:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829744.510565:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829744.510566:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829744.510567:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829744.510568:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829744.510569:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829744.510569:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829744.510570:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829744.510571:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829744.510572:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829744.510573:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829744.510574:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510575:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829744.510577:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829744.510578:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829744.510579:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829744.510581:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829745.020778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829745.020788:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829745.020792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829745.020799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829745.020804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829745.020808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829745.020812:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829745.020816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829745.020821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829745.212741:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298565292, at 4298565312
00000100:00000040:3.0:1787829745.212756:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298565292, at 4298565312
00000100:00000010:2.0:1787829745.212759:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00000100:00000010:3.0:1787829745.212774:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00010000:00000001:0.0:1787829745.340795:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829745.340804:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829745.340820:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829745.340821:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829745.340824:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829745.340827:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829745.340850:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829745.340852:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829745.340855:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829746.044787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829746.044796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829746.044798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829746.044803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829746.044806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829746.044810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829746.044813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829746.044816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829746.044818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829746.364807:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829746.364813:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829746.364818:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829746.364820:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829746.364823:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829746.364824:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829746.364898:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829746.364902:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829746.364905:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829747.068805:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829747.068812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829747.068814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829747.068820:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829747.068824:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829747.068828:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829747.068831:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829747.068834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829747.068837:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829747.388815:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829747.388823:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829747.388829:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829747.388830:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829747.388833:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829747.388834:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829747.388943:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829747.388947:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829747.388950:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829748.092878:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829748.092887:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829748.092889:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829748.092896:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829748.092900:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829748.092904:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829748.092906:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829748.092909:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829748.092913:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829748.412752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829748.412759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829748.412777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829748.412778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829748.412782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829748.412784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829748.412805:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829748.412807:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829748.412809:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829749.116722:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829749.116728:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829749.116730:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829749.116735:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829749.116737:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829749.116740:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.116742:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829749.116744:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.116746:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829749.436847:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829749.436854:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829749.436858:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829749.436860:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829749.436862:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829749.436864:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829749.436953:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829749.436959:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829749.436963:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829749.628803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829749.628811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829749.628813:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829749.628816:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829749.628820:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829749.628826:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3897
00000400:00000200:1.0:1787829749.628833:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829749.628839:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829749.628841:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829749.628844:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.628846:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.628847:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.628849:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829749.628852:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829749.628854:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829749.628855:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829749.628858:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829749.628860:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829749.628861:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829749.628863:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.628864:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.628866:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.628868:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829749.628870:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829749.628873:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829749.628874:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829749.628876:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829749.628879:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829749.628881:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.628882:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829749.628884:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829749.628892:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829749.628894:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829749.628897:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829749.628899:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.628901:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829749.628904:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829749.628908:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829749.628916:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829749.628923:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829749.628933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829749.628944:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829749.628949:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829749.628954:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829749.628977:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829749.628979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829749.628982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829749.628983:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829749.628984:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829749.628986:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829749.628988:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3897
00000400:00000200:1.0:1787829749.628991:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829749.628994:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829749.628995:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829749.628998:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.628999:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.629001:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629003:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829749.629005:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829749.629006:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829749.629007:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829749.629009:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829749.629010:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829749.629012:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829749.629014:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829749.629016:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829749.629016:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829749.629021:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787829749.629021:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629022:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829749.629025:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787829749.629026:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829749.629031:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:0.0:1787829749.629034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829749.629035:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829749.629041:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829749.629048:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829749.629049:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829749.629050:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829749.629056:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.629057:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829749.629060:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829749.629061:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829749.629063:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829749.629068:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327375232:192.168.203.116@tcp:250:kworker.0
00000008:00100000:1.0:1787829749.629068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829749.629076:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629077:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829749.629080:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.629081:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829749.629085:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.629087:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:1.0:1787829749.629087:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829749.629091:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000010:0.0:1787829749.629094:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829749.629095:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829749.629098:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.629099:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.629101:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829749.629103:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829749.629105:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829749.629106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829749.629111:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
02000000:00000010:1.0:1787829749.629112:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000400:00000010:0.0:1787829749.629117:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:1.0:1787829749.629117:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829749.629123:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829749.629124:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375232, portal 25
00000100:00000001:0.0:1787829749.629130:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829749.629133:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000001:1.0:1787829749.629135:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829749.629138:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829749.629141:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829749.629145:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629162:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787829749.629165:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829749.629167:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829749.629169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787829749.629171:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00080000:1.0:1787829749.629173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:0.0:1787829749.629175:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327375232, offset 0
00000100:00000001:1.0:1787829749.629176:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829749.629194:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829749.629199:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629200:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829749.629204:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3897
00000400:00000200:0.0:1787829749.629208:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.629214:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829749.629218:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829749.629222:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829749.629226:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829749.629229:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829749.629233:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40703:40703:256:4294967295] 192.168.203.116@tcp LPNI seq info [40703:40703:8:4294967295]
00000400:00000200:0.0:1787829749.629238:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829749.629244:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829749.629248:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829749.629253:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829749.629258:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787829749.629262:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629266:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.629270:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829749.629274:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629274:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829749.629277:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829749.629279:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829749.629280:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:0.0:1787829749.629282:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000020:00000040:1.0:1787829749.629282:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829749.629287:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829749.629290:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829749.629294:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829749.629295:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829749.629299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:1.0:1787829749.629299:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.629305:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629307:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829749.629311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:1.0:1787829749.629312:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787829749.629317:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00080000:1.0:1787829749.629318:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629324:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829749.629327:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000100:0.0:1787829749.629328:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:1.0:1787829749.629331:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:0.0:1787829749.629332:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
02000000:00000001:1.0:1787829749.629336:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.629338:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829749.629340:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:1.0:1787829749.629341:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829749.629345:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.629348:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829749.629349:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000010:1.0:1787829749.629353:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000400:00000200:0.0:1787829749.629354:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000020:00000040:1.0:1787829749.629357:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000010:0.0:1787829749.629358:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
02000000:00000001:1.0:1787829749.629362:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829749.629364:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:1.0:1787829749.629364:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.629366:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829749.629368:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1787829749.629371:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829749.629371:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829749.629377:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829749.629382:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629388:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:1.0:1787829749.629389:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829749.629395:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829749.629402:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829749.629406:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629411:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829749.629412:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629414:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829749.629421:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.629421:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829749.629427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829749.629430:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829749.629432:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829749.629434:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298570412
00000100:00000001:1.0:1787829749.629435:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829749.629437:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829749.629440:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:0.0:1787829749.629441:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc600.
00000100:00080000:1.0:1787829749.629442:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3897
00000400:00000200:1.0:1787829749.629448:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829749.629450:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000400:00000200:1.0:1787829749.629452:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829749.629455:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829749.629457:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629458:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.629459:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629460:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829749.629462:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829749.629462:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829749.629464:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298570412
00000020:00000001:1.0:1787829749.629464:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829749.629465:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000010:0.0:1787829749.629470:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcca80.
00000020:00000001:1.0:1787829749.629470:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829749.629473:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829749.629476:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000040:1.0:1787829749.629478:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629481:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829749.629482:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829749.629483:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829749.629484:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829749.629485:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829749.629489:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829749.629490:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829749.629493:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829749.629494:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829749.629497:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829749.629500:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1787829749.629503:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749]  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829749.629503:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829749.629508:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829749.629512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829749.629516:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829749.629517:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787829749.629518:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829749.629522:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000010:1.0:1787829749.629523:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000400:00000010:0.0:1787829749.629525:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000020:00000040:1.0:1787829749.629528:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000200:0.0:1787829749.629530:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
02000000:00000001:1.0:1787829749.629533:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1787829749.629535:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
02000000:00000001:1.0:1787829749.629536:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629538:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829749.629541:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829749.629544:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1787829749.629545:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829749.629547:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829749.629551:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829749.629556:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829749.629560:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829749.629563:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829749.629577:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829749.629578:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829749.629581:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629582:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829749.629585:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629587:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.629591:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829749.629594:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829749.629599:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3907)
00000100:00100000:0.0:1787829749.629601:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629612:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829749.629618:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629627:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829749.629630:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.629631:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.629632:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.629634:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829749.629635:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829749.629637:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829749.629640:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829749.629642:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829749.629645:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829749.629650:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327375232:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829749.629680:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.629683:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.629687:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327375232/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629701:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829749.629704:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829749.629707:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829749.629711:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829749.629713:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829749.629714:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829749.629717:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829749.629719:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:0.0:1787829749.629724:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829749.629726:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829749.629728:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829749.629731:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829749.629733:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.629735:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.629742:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829749.629745:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629747:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829749.629749:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829749.629750:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829749.629755:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629763:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829749.629764:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.629767:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829749.629769:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829749.629772:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829749.629775:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375360:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829749.629778:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829749.629780:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829749.629781:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.629782:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829749.629785:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829749.629787:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829749.629790:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829749.629792:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829749.629794:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829749.629797:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375360, portal 10
00000100:00000001:0.0:1787829749.629798:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829749.629799:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829749.629804:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629810:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829749.629813:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829749.629816:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829749.629818:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327375360, offset 0
00000400:00000200:0.0:1787829749.629823:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829749.629828:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.629831:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829749.629835:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829749.629838:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829749.629841:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829749.629844:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829749.629847:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40704:40704:256:4294967295] 192.168.203.116@tcp LPNI seq info [40704:40704:8:4294967295]
00000400:00000200:0.0:1787829749.629861:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.629867:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829749.629870:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829749.629874:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829749.629876:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829749.629878:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829749.629882:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829749.629883:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829749.629886:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829749.629887:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829749.629889:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829749.629892:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829749.629894:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829749.629896:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829749.629897:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829749.629900:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629906:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.629908:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629912:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629913:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829749.629916:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829749.629920:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829749.629923:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829749.629925:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298569739
00000100:00000001:0.0:1787829749.629928:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298570412
00000100:00000001:0.0:1787829749.629930:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298569739
00000100:00000001:0.0:1787829749.629933:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298570412
00000100:00000001:0.0:1787829749.629935:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829749.629937:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.629939:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829749.629943:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749]  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829749.629952:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829749.629954:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829749.629956:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829749.629958:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829749.629960:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829749.629963:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829749.629966:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829749.629974:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629979:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829749.629980:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.629981:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.629982:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829749.629985:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629989:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829749.629992:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.629997:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829749.629999:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630000:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.630000:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630002:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829749.630003:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829749.630005:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829749.630007:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829749.630008:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829749.630010:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829749.630015:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375360:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829749.630018:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829749.630020:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630025:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829749.630025:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.630027:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829749.630028:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829749.630029:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829749.630031:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375488:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829749.630033:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829749.630034:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829749.630036:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.630037:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829749.630040:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829749.630042:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829749.630044:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829749.630046:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829749.630049:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829749.630051:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375488, portal 4
00000100:00000001:0.0:1787829749.630052:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829749.630053:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829749.630057:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630062:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829749.630063:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829749.630065:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829749.630066:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327375488, offset 0
00000400:00000200:0.0:1787829749.630069:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829749.630072:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.630074:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829749.630076:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829749.630078:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829749.630080:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829749.630081:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829749.630084:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40705:40705:256:4294967295] 192.168.203.116@tcp LPNI seq info [40705:40705:8:4294967295]
00000400:00000200:0.0:1787829749.630086:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.630090:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829749.630093:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829749.630096:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829749.630098:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829749.630100:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829749.630103:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829749.630105:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829749.630108:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829749.630109:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829749.630111:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829749.630114:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829749.630116:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829749.630117:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829749.630119:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829749.630121:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630126:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.630128:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630132:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.630133:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829749.630135:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829749.630137:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829749.630139:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829749.630140:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298569746
00000100:00000001:0.0:1787829749.630142:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298570412
00000100:00000001:0.0:1787829749.630144:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298569746
00000100:00000001:0.0:1787829749.630145:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298570412
00000100:00000001:0.0:1787829749.630146:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829749.630148:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.630149:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829749.630151:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749]  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829749.630155:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829749.630156:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829749.630157:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829749.630158:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829749.630159:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829749.630160:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829749.630162:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829749.630167:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630176:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829749.630177:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630179:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.630180:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829749.630198:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630202:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829749.630205:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630210:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829749.630211:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630212:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.630213:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630214:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829749.630215:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829749.630217:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829749.630219:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829749.630220:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829749.630222:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829749.630230:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375488:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829749.630235:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829749.630238:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630245:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829749.630246:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.630249:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829749.630251:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829749.630253:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829749.630256:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375616:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829749.630259:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829749.630262:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829749.630263:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829749.630265:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829749.630269:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829749.630271:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829749.630275:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829749.630278:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829749.630281:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829749.630283:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375616, portal 4
00000100:00000001:0.0:1787829749.630285:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829749.630286:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:0.0:1787829749.630291:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630295:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829749.630296:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829749.630298:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000100:00000200:0.0:1787829749.630300:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327375616, offset 0
00000400:00000200:0.0:1787829749.630302:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829749.630305:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.630308:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829749.630310:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829749.630312:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829749.630313:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829749.630315:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829749.630317:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40706:40706:256:4294967295] 192.168.203.116@tcp LPNI seq info [40706:40706:8:4294967295]
00000400:00000200:0.0:1787829749.630320:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829749.630324:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829749.630328:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829749.630330:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829749.630332:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829749.630334:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829749.630337:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829749.630339:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829749.630341:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829749.630343:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:0.0:1787829749.630345:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829749.630347:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829749.630349:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:0.0:1787829749.630350:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:0.0:1787829749.630352:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829749.630354:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630359:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.630361:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630366:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.630367:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829749.630368:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829749.630371:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829749.630373:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829749.630374:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298569751
00000100:00000001:0.0:1787829749.630376:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298570412
00000100:00000001:0.0:1787829749.630377:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298569751
00000100:00000001:0.0:1787829749.630378:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298570412
00000100:00000001:0.0:1787829749.630380:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829749.630381:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.630382:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829749.630384:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749]  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829749.630390:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829749.630391:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829749.630392:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829749.630394:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829749.630395:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829749.630396:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829749.630398:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829749.630403:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630407:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829749.630408:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630409:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.630410:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829749.630412:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630416:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829749.630418:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630422:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829749.630424:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630425:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829749.630426:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630427:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829749.630428:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829749.630430:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829749.630432:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829749.630433:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829749.630436:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829749.630441:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375616:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829749.630443:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630445:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.630446:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327375360/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630450:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829749.630451:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829749.630452:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829749.630454:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829749.630455:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829749.630456:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829749.630457:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829749.630458:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829749.630460:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829749.630461:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829749.630461:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829749.630464:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829749.630465:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630466:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.630468:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327375488/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630472:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829749.630472:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829749.630473:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829749.630474:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829749.630475:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829749.630476:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829749.630477:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829749.630478:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829749.630479:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829749.630480:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829749.630481:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829749.630482:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829749.630483:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630484:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829749.630486:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327375616/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829749.630490:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829749.630491:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829749.630491:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829749.630509:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829749.630510:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829749.630511:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829749.630513:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829749.630514:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829749.630516:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829749.630516:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829749.630518:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829749.630519:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829749.630520:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630522:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829749.630526:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829749.630527:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829749.630529:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829749.630530:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829750.140784:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829750.140791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829750.140793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829750.140799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829750.140802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829750.140806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829750.140809:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829750.140812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829750.140815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829750.332756:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298570412, at 4298570432
00000100:00000010:3.0:1787829750.332769:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00000100:00000040:2.0:1787829750.332785:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298570412, at 4298570432
00000100:00000010:2.0:1787829750.332800:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00010000:00000001:0.0:1787829750.460731:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829750.460738:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829750.460744:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829750.460748:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829750.460753:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829750.460758:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829750.460760:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829750.460763:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829750.460766:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829750.460769:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829750.460771:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829750.460773:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829750.460819:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829750.460826:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829750.460831:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829751.164774:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829751.164782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829751.164784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829751.164790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829751.164794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829751.164797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829751.164800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829751.164803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829751.164806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829751.484805:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829751.484815:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829751.484825:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829751.484827:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829751.484833:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829751.484835:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829751.484867:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829751.484869:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829751.484872:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829752.188764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829752.188771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829752.188773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829752.188777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829752.188780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829752.188782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829752.188784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829752.188786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829752.188788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829752.508758:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829752.508764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829752.508769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829752.508771:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829752.508774:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829752.508774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829752.508854:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829752.508861:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829752.508867:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829753.212739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829753.212744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829753.212746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829753.212750:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829753.212753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829753.212757:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829753.212759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829753.212761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829753.212763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829753.532763:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829753.532770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829753.532776:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829753.532778:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829753.532782:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829753.532783:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829753.532820:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829753.532823:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829753.532826:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829754.236769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829754.236775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829754.236778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829754.236783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829754.236787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829754.236791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829754.236795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829754.236797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829754.236800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556767:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829754.556774:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829754.556776:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829754.556778:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556782:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829754.556784:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556786:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556788:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829754.556792:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829754.556793:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829754.556794:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556795:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829754.556796:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556798:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556799:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829754.556801:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829754.556802:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829754.556817:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829754.556819:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829754.556821:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829754.748797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829754.748805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829754.748807:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829754.748810:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829754.748814:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829754.748820:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3902
00000400:00000200:3.0:1787829754.748827:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829754.748832:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829754.748834:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829754.748837:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.748839:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829754.748841:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.748843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829754.748845:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829754.748847:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829754.748849:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829754.748851:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829754.748853:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829754.748855:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829754.748857:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.748860:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829754.748861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.748864:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829754.748867:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829754.748871:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829754.748874:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829754.748878:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829754.748882:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829754.748885:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.748887:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829754.748890:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829754.748900:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829754.748905:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829754.748909:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829754.748912:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.748915:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829754.748918:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829754.748922:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829754.748929:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829754.748935:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829754.748944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829754.748956:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829754.748960:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829754.748965:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829754.748993:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829754.748995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829754.748998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829754.748999:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829754.749001:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829754.749003:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829754.749005:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3902
00000400:00000200:3.0:1787829754.749009:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829754.749011:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829754.749013:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829754.749015:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749017:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829754.749019:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749020:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829754.749023:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829754.749024:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829754.749025:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829754.749027:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829754.749028:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829754.749030:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829754.749032:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749033:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829754.749035:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787829754.749036:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749040:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829754.749042:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1787829754.749043:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829754.749046:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:3.0:1787829754.749047:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749061:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829754.749064:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829754.749066:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829754.749069:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829754.749074:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749078:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829754.749083:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:3.0:1787829754.749083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829754.749087:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787829754.749087:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749091:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749092:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:3.0:1787829754.749095:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
02000000:00000001:0.0:1787829754.749096:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829754.749101:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787829754.749101:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829754.749105:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:0.0:1787829754.749108:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327375744:192.168.203.116@tcp:250:kworker.0
02000000:00000001:3.0:1787829754.749109:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749113:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829754.749116:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787829754.749116:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829754.749118:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749121:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:3.0:1787829754.749123:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
02000000:00000001:0.0:1787829754.749125:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749129:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749130:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829754.749162:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
00000100:00100000:3.0:1787829754.749162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829754.749169:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749174:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829754.749177:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749182:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829754.749182:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829754.749190:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000100:00000001:3.0:1787829754.749192:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829754.749197:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829754.749210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829754.749211:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829754.749212:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:0.0:1787829754.749213:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:3.0:1787829754.749215:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829754.749218:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3902
00000100:00000200:0.0:1787829754.749221:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375744, portal 25
00000400:00000200:3.0:1787829754.749222:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829754.749227:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000400:00000200:3.0:1787829754.749228:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829754.749230:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00080000:3.0:1787829754.749231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829754.749236:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749239:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829754.749242:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829754.749243:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749246:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829754.749250:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829754.749252:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829754.749255:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829754.749257:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829754.749261:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:0.0:1787829754.749263:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829754.749276:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829754.749277:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef2a8.
00000020:00000001:3.0:1787829754.749279:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:0.0:1787829754.749281:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327375744, offset 0
00000100:00000040:3.0:1787829754.749288:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829754.749291:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749292:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829754.749296:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749297:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829754.749299:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829754.749301:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749303:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829754.749304:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829754.749307:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829754.749310:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829754.749311:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749314:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829754.749316:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829754.749319:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749321:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829754.749322:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749328:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:3.0:1787829754.749340:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000400:00000200:0.0:1787829754.749343:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000040:3.0:1787829754.749344:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829754.749349:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787829754.749350:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829754.749353:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749356:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749357:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829754.749360:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829754.749363:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40707:40707:256:4294967295] 192.168.203.116@tcp LPNI seq info [40707:40707:8:4294967295]
02000000:00000001:3.0:1787829754.749363:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829754.749367:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000400:00000200:0.0:1787829754.749370:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829754.749372:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829754.749378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829754.749387:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787829754.749393:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829754.749406:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829754.749410:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:3.0:1787829754.749414:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1787829754.749417:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000100:00000001:3.0:1787829754.749425:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829754.749429:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829754.749429:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829754.749436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829754.749440:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1787829754.749441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:3.0:1787829754.749442:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829754.749448:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829754.749450:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3902
00000400:00000200:3.0:1787829754.749453:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829754.749455:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829754.749457:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829754.749459:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829754.749460:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749474:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829754.749479:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829754.749481:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829754.749482:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829754.749487:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829754.749488:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829754.749489:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298575532
00000020:00000001:3.0:1787829754.749490:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829754.749493:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829754.749497:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829754.749499:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000010:0.0:1787829754.749500:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc9c0.
00000100:00000040:3.0:1787829754.749502:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749506:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829754.749511:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829754.749513:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787829754.749513:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829754.749515:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829754.749520:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829754.749522:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829754.749525:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829754.749526:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298575532
02000000:00000001:3.0:1787829754.749528:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829754.749532:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829754.749533:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc600.
00000100:00000001:3.0:1787829754.749534:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829754.749536:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829754.749539:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749541:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787829754.749541:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829754.749548:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829754.749551:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829754.749554:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829754.749555:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829754.749557:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749560:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829754.749560:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829754.749564:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829754.749567:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829754.749569:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749574:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1787829754.749576:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829754.749580:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829754.749583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829754.749584:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829754.749588:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787829754.749591:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829754.749595:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787829754.749595:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749600:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787829754.749602:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829754.749608:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829754.749611:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3912)
00000100:00000001:0.0:1787829754.749612:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829754.749622:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829754.749628:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829754.749630:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749632:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829754.749634:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829754.749635:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829754.749638:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375872:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829754.749640:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829754.749648:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829754.749650:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749653:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829754.749702:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829754.749704:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749708:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829754.749712:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708d68.
00000400:00000010:0.0:1787829754.749716:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef330.
00000100:00000200:0.0:1787829754.749720:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327375872, portal 10
00000100:00000001:0.0:1787829754.749723:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829754.749725:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:0.0:1787829754.749733:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829754.749741:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829754.749745:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829754.749749:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef880.
00000100:00000200:0.0:1787829754.749752:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327375872, offset 0
00000400:00000200:0.0:1787829754.749760:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829754.749766:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829754.749770:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829754.749775:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829754.749778:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829754.749780:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829754.749783:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829754.749786:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40708:40708:255:4294967295] 192.168.203.116@tcp LPNI seq info [40708:40708:7:4294967295]
00000400:00000200:0.0:1787829754.749791:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829754.749798:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829754.749801:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829754.749804:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829754.749808:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829754.749811:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829754.749813:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829754.749822:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829754.749825:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829754.749827:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298574859
00000100:00000001:0.0:1787829754.749829:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298575532
00000100:00000001:0.0:1787829754.749830:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298574859
00000100:00000001:0.0:1787829754.749832:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298575532
00000100:00000001:0.0:1787829754.749833:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829754.749835:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749836:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829754.749840:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829754.749844:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829754.749845:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749847:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829754.749848:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829754.749850:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829754.749852:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376000:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829754.749854:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829754.749856:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829754.749857:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749858:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829754.749860:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829754.749862:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749864:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829754.749866:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829754.749868:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829754.749870:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376000, portal 4
00000100:00000001:0.0:1787829754.749872:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829754.749873:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000100:00000040:0.0:1787829754.749877:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829754.749882:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829754.749883:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829754.749885:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef550.
00000100:00000200:0.0:1787829754.749887:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327376000, offset 0
00000400:00000200:0.0:1787829754.749889:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829754.749891:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829754.749894:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829754.749896:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829754.749897:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829754.749899:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829754.749901:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829754.749903:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40709:40709:254:4294967295] 192.168.203.116@tcp LPNI seq info [40709:40709:6:4294967295]
00000400:00000200:0.0:1787829754.749906:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829754.749910:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829754.749912:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829754.749915:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7f00.
00000800:00000200:0.0:1787829754.749917:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829754.749920:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829754.749922:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829754.749923:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829754.749926:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829754.749927:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298574866
00000100:00000001:0.0:1787829754.749929:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298575532
00000100:00000001:0.0:1787829754.749930:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298574866
00000100:00000001:0.0:1787829754.749931:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298575532
00000100:00000001:0.0:1787829754.749939:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829754.749940:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749942:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.749943:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829754.749946:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829754.749954:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829754.749955:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829754.749956:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829754.749957:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829754.749960:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829754.749965:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829754.749966:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749967:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829754.749969:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829754.749970:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829754.749972:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376128:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829754.749974:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829754.749976:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829754.749977:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829754.749979:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829754.749982:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829754.749984:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829754.749987:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829754.749990:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829754.749993:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829754.749996:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376128, portal 4
00000100:00000001:0.0:1787829754.749998:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829754.750000:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829754.750006:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829754.750012:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829754.750014:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829754.750017:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fefee0.
00000100:00000200:0.0:1787829754.750019:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327376128, offset 0
00000400:00000200:0.0:1787829754.750023:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829754.750027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829754.750031:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829754.750034:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829754.750037:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829754.750039:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829754.750042:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829754.750045:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40710:40710:253:4294967295] 192.168.203.116@tcp LPNI seq info [40710:40710:5:4294967295]
00000400:00000200:0.0:1787829754.750049:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829754.750055:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829754.750060:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829754.750063:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7000.
00000800:00000200:0.0:1787829754.750066:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829754.750070:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829754.750073:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829754.750075:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829754.750078:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829754.750080:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298574871
00000100:00000001:0.0:1787829754.750083:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298575532
00000100:00000001:0.0:1787829754.750085:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298574871
00000100:00000001:0.0:1787829754.750087:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298575532
00000100:00000001:0.0:1787829754.750089:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829754.750091:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.750094:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.750097:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829754.750100:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829754.750102:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829754.750104:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829754.750105:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829754.750107:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829754.750109:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829755.260761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829755.260771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829755.260774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829755.260782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829755.260787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829755.260793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829755.260797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829755.260801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829755.260805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829755.452801:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298575532, at 4298575552
00000100:00000040:3.0:1787829755.452802:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298575532, at 4298575552
00000100:00000010:3.0:1787829755.452821:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:2.0:1787829755.452823:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00010000:00000001:0.0:1787829755.580800:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829755.580808:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829755.580814:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829755.580815:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829755.580818:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829755.580819:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829755.580887:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829755.580892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829755.580895:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829756.284766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829756.284773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829756.284776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829756.284781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829756.284785:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829756.284789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829756.284791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829756.284793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829756.284796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829756.604800:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829756.604805:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829756.604810:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829756.604811:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829756.604814:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829756.604816:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829756.604852:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829756.604855:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829756.604858:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829757.308760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829757.308765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829757.308767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829757.308770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829757.308773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829757.308776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.308778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829757.308780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.308782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829757.628746:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829757.628751:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829757.628755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829757.628756:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829757.628759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829757.628760:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829757.628775:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829757.628777:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829757.628779:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829757.821014:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829757.821064:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829757.821074:0:2337:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829757.821077:0:2337:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829757.821083:0:2337:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829757.821087:0:2337:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829757.821091:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829757.821096:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:2.0:1787829757.821101:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829757.821105:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829757.821112:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef2a8
00000400:00000010:2.0:1787829757.821114:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef2a8.
00000100:00000001:2.0:1787829757.821120:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829757.821150:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829757.821184:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829757.821189:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829757.821199:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829757.821201:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829757.821214:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829757.821219:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:2.0:1787829757.821225:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829757.821231:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829757.821235:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef880
00000400:00000010:2.0:1787829757.821238:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef880.
00000100:00000001:2.0:1787829757.821242:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829757.821251:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821257:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829757.821268:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829757.821276:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787829757.821277:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787829757.821287:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829757.821301:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829757]  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829757.821303:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829757.821312:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000001:0.0:1787829757.821331:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829757.821336:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000800:00000100:2.0:1787829757.821339:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000400:00000010:0.0:1787829757.821343:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708d68.
00000800:00000010:2.0:1787829757.821346:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000200:0.0:1787829757.821354:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef330
00000400:00000200:2.0:1787829757.821355:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000010:0.0:1787829757.821364:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef330.
00000400:00000200:2.0:1787829757.821368:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829757.821379:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef550
00000100:00000001:0.0:1787829757.821380:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000010:2.0:1787829757.821389:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef550.
00000100:00000001:2.0:1787829757.821397:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829757.821398:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787829757.821411:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829757.821427:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829757.821433:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829757.821451:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821453:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829757.821459:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829757.821472:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829757.821479:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829757.821479:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829757.821490:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:2.0:1787829757.821496:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829757.821502:0:2337:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000100:00100000:0.0:1787829757.821509:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829757.821510:0:2337:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:0.0:1787829757.821529:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1787829757.821530:0:2337:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829757.821543:0:2337:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefee0
00000100:00000040:0.0:1787829757.821551:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787829757.821551:0:2337:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefee0.
00000100:00000001:2.0:1787829757.821560:0:2337:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829757.821573:0:2337:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821592:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829757.821608:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829757.821610:0:2337:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787829757.821617:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1787829757.821618:0:2337:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821622:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821633:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829757.821638:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829757.821641:0:2337:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829757.821646:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1787829757.821647:0:2337:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1787829757.821696:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829757.821702:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829757.821709:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829757.821721:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327375872:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829757.821728:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829757.821733:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829757]  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829757.821743:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829757.821745:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829757.821749:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829757.821753:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829757.821755:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829757.821760:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829757.821764:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829757.821777:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821784:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829757.821786:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821788:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829757.821790:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829757.821796:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821803:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829757.821808:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821815:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829757.821818:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821819:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829757.821821:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821822:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829757.821824:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829757.821827:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829757.821831:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829757.821833:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829757.821837:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829757.821844:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376000:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829757.821849:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829757.821855:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829757]  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829757.821862:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829757.821863:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829757.821865:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829757.821867:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829757.821869:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829757.821882:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829757.821886:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829757.821894:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821900:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829757.821902:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821903:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829757.821905:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829757.821909:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821917:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829757.821921:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821929:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829757.821932:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821934:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829757.821935:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.821936:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829757.821938:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829757.821941:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829757.821945:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829757.821947:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829757.821952:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829757.821959:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327375744:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829757.821963:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829757.821965:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829757.821969:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327375872/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.821976:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829757.821981:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829757.821985:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829757.821990:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829757.821992:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829757.821994:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829757.821997:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829757.822000:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:0.0:1787829757.822004:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829757.822007:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829757.822009:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:0.0:1787829757.822014:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829757.822016:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822018:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829757.822022:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327376000/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.822029:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829757.822034:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829757.822035:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829757.822039:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829757.822040:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829757.822041:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829757.822044:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829757.822045:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:0.0:1787829757.822048:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829757.822050:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829757.822051:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:0.0:1787829757.822054:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829757.822055:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822057:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829757.822060:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327375744/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.822066:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829757.822068:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829757.822069:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829757.822072:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829757.822073:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829757.822075:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829757.822077:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829757.822079:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:0.0:1787829757.822081:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829757.822082:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829757.822084:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:0.0:1787829757.822086:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829757.822087:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822090:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829757.822093:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829757.822094:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829757.822096:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829757.822100:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829757]  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829757.822109:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829757.822110:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829757.822112:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829757.822115:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829757.822117:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829757.822119:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829757.822122:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829757.822150:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.822157:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829757.822160:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822162:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829757.822163:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829757.822167:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.822174:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829757.822179:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.822186:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829757.822188:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822190:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829757.822191:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822193:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829757.822195:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829757.822197:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829757.822201:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829757.822203:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829757.822207:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829757.822215:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376128:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829757.822219:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822221:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829757.822223:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327376128/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829757.822230:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829757.822232:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829757.822234:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829757.822237:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829757.822239:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829757.822240:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829757.822242:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829757.822244:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:0.0:1787829757.822247:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829757.822248:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829757.822250:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829757.822252:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829757.822253:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822254:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829757.822261:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829757.822262:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829757.822265:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829757.822266:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829758.332740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829758.332745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829758.332747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829758.332750:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829758.332753:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829758.332756:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829758.332758:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829758.332760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829758.332762:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829758.652742:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829758.652748:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829758.652753:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829758.652755:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829758.652758:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829758.652759:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829758.652822:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829758.652827:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829758.652831:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829759.356726:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829759.356737:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829759.356740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829759.356746:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829759.356750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829759.356754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.356756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829759.356760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.356762:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829759.676842:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829759.676847:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829759.676851:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829759.676852:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829759.676854:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829759.676855:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829759.676902:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829759.676904:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829759.676907:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829759.868738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829759.868746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829759.868748:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829759.868751:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829759.868755:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829759.868761:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3907
00000400:00000200:3.0:1787829759.868768:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829759.868773:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829759.868776:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829759.868778:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868781:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829759.868783:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868786:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829759.868789:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829759.868791:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829759.868793:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829759.868796:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829759.868798:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829759.868800:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829759.868803:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868805:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829759.868809:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868810:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829759.868813:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829759.868815:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868816:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829759.868818:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829759.868821:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829759.868823:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829759.868825:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829759.868826:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829759.868834:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829759.868837:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829759.868840:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829759.868842:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829759.868844:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829759.868846:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829759.868850:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829759.868856:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829759.868861:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829759.868869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829759.868876:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829759.868880:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829759.868883:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829759.868907:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829759.868911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829759.868914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829759.868915:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829759.868917:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829759.868920:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829759.868923:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3907
00000400:00000200:3.0:1787829759.868926:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829759.868930:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829759.868933:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829759.868936:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868938:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829759.868941:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868944:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829759.868947:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829759.868949:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829759.868951:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829759.868954:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829759.868956:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829759.868959:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829759.868962:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829759.868964:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787829759.868964:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829759.868969:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829759.868972:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829759.868972:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829759.868977:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787829759.868977:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829759.868979:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829759.868981:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829759.868983:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829759.868988:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829759.868992:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829759.868995:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829759.868996:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829759.869002:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829759.869009:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869012:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787829759.869013:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869016:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829759.869020:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
02000000:00000001:0.0:1787829759.869022:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1787829759.869025:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829759.869027:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829759.869032:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829759.869033:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829759.869038:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829759.869039:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327376256:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829759.869041:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829759.869046:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829759.869047:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787829759.869050:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829759.869058:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829759.869062:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829759.869070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829759.869078:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829759.869079:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787829759.869081:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.869084:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829759.869089:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829759.869090:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829759.869099:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829759.869104:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.869132:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829759.869135:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829759.869139:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00080000:3.0:1787829759.869140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829759.869147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829759.869149:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1787829759.869151:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00080000:3.0:1787829759.869152:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829759.869157:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:0.0:1787829759.869161:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376256, portal 25
00000100:00080000:3.0:1787829759.869162:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3907
00000100:00000001:0.0:1787829759.869166:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829759.869169:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000400:00000200:3.0:1787829759.869172:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829759.869177:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829759.869179:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829759.869182:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829759.869185:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869190:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829759.869194:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869199:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829759.869204:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:3.0:1787829759.869205:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829759.869209:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000040:0.0:1787829759.869212:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829759.869213:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1787829759.869217:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000020:00000040:3.0:1787829759.869218:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829759.869223:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:0.0:1787829759.869224:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327376256, offset 0
00000020:00000001:3.0:1787829759.869227:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829759.869234:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829759.869239:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869240:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829759.869247:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869250:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829759.869253:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:3.0:1787829759.869256:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829759.869263:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829759.869270:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1787829759.869272:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869278:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.869279:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:3.0:1787829759.869284:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829759.869287:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787829759.869291:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829759.869294:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:3.0:1787829759.869297:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829759.869302:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829759.869303:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40711:40711:256:4294967295] 192.168.203.116@tcp LPNI seq info [40711:40711:8:4294967295]
00000008:00100000:3.0:1787829759.869308:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829759.869311:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00000001:3.0:1787829759.869315:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829759.869319:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829759.869324:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000400:00000200:0.0:1787829759.869327:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1787829759.869329:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829759.869336:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.0:1787829759.869338:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829759.869339:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829759.869344:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829759.869345:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:0.0:1787829759.869347:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
02000000:00000001:3.0:1787829759.869348:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829759.869359:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829759.869363:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829759.869369:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000010:3.0:1787829759.869370:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000800:00000200:0.1:1787829759.869375:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000001:3.0:1787829759.869377:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829759.869383:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00100000:3.0:1787829759.869388:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1787829759.869389:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829759.869398:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787829759.869401:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829759.869406:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829759.869412:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829759.869420:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.869422:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787829759.869424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829759.869430:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00080000:3.0:1787829759.869430:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829759.869432:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1787829759.869436:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00080000:3.0:1787829759.869437:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829759.869444:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829759.869446:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1787829759.869447:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3907
00000100:00000200:0.0:1787829759.869454:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787829759.869456:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829759.869464:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829759.869469:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829759.869474:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787829759.869475:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869481:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829759.869482:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829759.869486:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.869493:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829759.869498:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829759.869502:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829759.869503:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869506:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829759.869514:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.869519:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829759.869521:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298580652
00000100:00000010:0.0:1787829759.869526:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcca80.
00000100:00000001:0.0:1787829759.869534:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829759.869542:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298580652
00000100:00000010:0.0:1787829759.869544:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc0c0.
00000100:00000001:0.0:1787829759.869548:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829759.869551:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829759.869553:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869556:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829759.869561:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759]  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829759.869569:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829759.869571:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829759.869573:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829759.869576:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829759.869578:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829759.869580:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829759.869583:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829759.869591:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869598:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829759.869600:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.869603:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.869605:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829759.869609:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869615:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829759.869620:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869627:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829759.869631:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.869632:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.869634:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.869636:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829759.869639:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829759.869642:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829759.869646:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829759.869649:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829759.869653:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829759.869698:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327376256:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829759.869703:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.869706:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.869709:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327376256/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869715:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829759.869717:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829759.869721:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829759.869724:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829759.869727:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829759.869728:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829759.869731:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829759.869734:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:0.0:1787829759.869738:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829759.869740:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829759.869746:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829759.869750:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829759.869751:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.869753:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.869761:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829759.869763:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869765:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829759.869767:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829759.869769:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829759.869773:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869780:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829759.869781:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.869783:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829759.869785:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829759.869788:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829759.869791:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376384:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829759.869794:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829759.869796:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829759.869798:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.869800:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829759.869803:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829759.869806:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.869809:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829759.869812:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829759.869816:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829759.869819:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376384, portal 10
00000100:00000001:0.0:1787829759.869821:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829759.869822:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:0.0:1787829759.869830:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869836:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829759.869839:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829759.869842:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829759.869851:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327376384, offset 0
00000400:00000200:0.0:1787829759.869856:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829759.869860:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829759.869865:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829759.869868:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829759.869871:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829759.869873:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829759.869876:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829759.869879:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40712:40712:256:4294967295] 192.168.203.116@tcp LPNI seq info [40712:40712:8:4294967295]
00000400:00000200:0.0:1787829759.869883:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829759.869890:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829759.869894:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829759.869898:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829759.869902:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829759.869905:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829759.869910:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829759.869912:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829759.869916:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829759.869918:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829759.869921:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829759.869925:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829759.869929:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829759.869931:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829759.869934:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829759.869937:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869946:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.869949:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.869954:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869956:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829759.869959:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829759.869963:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829759.869967:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829759.869969:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298579978
00000100:00000001:0.0:1787829759.869972:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298580652
00000100:00000001:0.0:1787829759.869975:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298579978
00000100:00000001:0.0:1787829759.869977:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298580652
00000100:00000001:0.0:1787829759.869980:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829759.869982:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.869984:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829759.869989:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759]  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829759.869997:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829759.869998:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829759.870000:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829759.870003:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829759.870005:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829759.870007:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829759.870010:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829759.870018:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870025:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829759.870027:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1787829759.870029:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829759.870030:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.870034:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1787829759.870035:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00100000:0.0:1787829759.870040:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829759.870041:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829759.870048:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829759.870057:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829759.870058:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.870064:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829759.870065:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787829759.870072:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829759.870079:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829759.870081:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:3.0:1787829759.870083:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829759.870086:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787829759.870090:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829759.870091:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.870095:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829759.870097:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870098:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:3.0:1787829759.870101:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829759.870103:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829759.870107:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1787829759.870108:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829759.870124:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829759.870130:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787829759.870130:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829759.870135:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829759.870136:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000008:00000001:3.0:1787829759.870143:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829759.870147:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829759.870148:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829759.870155:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000100:00100000:0.0:1787829759.870160:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376384:192.168.203.116@tcp:38:kworker.0
00000020:00000040:3.0:1787829759.870162:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829759.870169:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1787829759.870170:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829759.870174:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829759.870177:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829759.870179:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829759.870182:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829759.870185:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870192:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000010:3.0:1787829759.870194:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:0.0:1787829759.870195:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.870201:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787829759.870203:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.870205:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829759.870209:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829759.870215:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829759.870216:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376512:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829759.870225:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829759.870229:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829759.870231:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.870233:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829759.870237:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
00000100:00000001:3.0:1787829759.870240:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829759.870241:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829759.870246:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.870251:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829759.870258:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000040:3.0:1787829759.870258:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829759.870264:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829759.870270:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376512, portal 4
00000100:00000001:0.0:1787829759.870277:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829759.870279:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000001:3.0:1787829759.870279:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829759.870289:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3917)
00000100:00000040:0.0:1787829759.870290:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870297:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829759.870299:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829759.870301:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829759.870303:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327376512, offset 0
00000400:00000200:0.0:1787829759.870306:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829759.870309:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829759.870313:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829759.870316:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829759.870319:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829759.870320:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829759.870322:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829759.870324:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40713:40713:256:4294967295] 192.168.203.116@tcp LPNI seq info [40713:40713:8:4294967295]
00000400:00000200:0.0:1787829759.870327:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829759.870331:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829759.870335:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829759.870337:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829759.870339:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829759.870342:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829759.870345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829759.870348:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829759.870352:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829759.870354:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829759.870357:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829759.870361:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829759.870364:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829759.870366:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829759.870369:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829759.870372:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870380:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.870384:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870391:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870393:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829759.870396:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829759.870399:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829759.870403:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829759.870405:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298579986
00000100:00000001:0.0:1787829759.870407:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298580652
00000100:00000001:0.0:1787829759.870410:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298579986
00000100:00000001:0.0:1787829759.870412:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298580652
00000100:00000001:0.0:1787829759.870414:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829759.870416:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870417:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829759.870420:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759]  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829759.870426:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829759.870427:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829759.870428:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829759.870430:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829759.870431:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829759.870432:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829759.870434:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829759.870440:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870444:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829759.870445:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870446:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.870447:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829759.870450:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870455:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829759.870457:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870462:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829759.870463:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870464:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.870465:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870466:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829759.870467:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829759.870469:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829759.870471:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829759.870472:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829759.870476:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829759.870484:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376512:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829759.870488:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870490:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.870494:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327376384/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870500:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829759.870502:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829759.870504:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829759.870507:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829759.870509:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829759.870510:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829759.870512:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829759.870514:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:0.0:1787829759.870517:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829759.870518:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829759.870520:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829759.870522:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829759.870523:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870525:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.870529:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327376512/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870535:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829759.870537:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829759.870538:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829759.870541:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829759.870542:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829759.870543:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829759.870545:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829759.870547:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:0.0:1787829759.870550:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829759.870551:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829759.870552:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:0.0:1787829759.870556:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829759.870557:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870559:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870563:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829759.870565:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870567:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829759.870569:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829759.870570:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829759.870574:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870578:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829759.870580:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.870581:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829759.870583:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829759.870584:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829759.870586:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376640:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829759.870589:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829759.870590:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829759.870591:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829759.870593:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829759.870595:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829759.870596:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829759.870599:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829759.870600:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829759.870603:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829759.870605:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376640, portal 4
00000100:00000001:0.0:1787829759.870606:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829759.870607:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:0.0:1787829759.870611:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870616:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829759.870617:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829759.870619:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829759.870620:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327376640, offset 0
00000400:00000200:0.0:1787829759.870623:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829759.870625:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829759.870628:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829759.870630:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829759.870632:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829759.870633:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829759.870635:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829759.870637:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40714:40714:256:4294967295] 192.168.203.116@tcp LPNI seq info [40714:40714:8:4294967295]
00000400:00000200:0.0:1787829759.870640:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829759.870644:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829759.870647:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829759.870649:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829759.870652:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829759.870654:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829759.870674:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829759.870676:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829759.870679:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829759.870680:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829759.870683:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829759.870685:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829759.870687:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829759.870688:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829759.870690:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829759.870692:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870698:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.870699:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870704:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870705:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829759.870707:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829759.870709:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829759.870711:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829759.870712:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298579992
00000100:00000001:0.0:1787829759.870714:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298580652
00000100:00000001:0.0:1787829759.870715:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298579992
00000100:00000001:0.0:1787829759.870717:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298580652
00000100:00000001:0.0:1787829759.870718:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829759.870719:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870721:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829759.870723:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759]  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829759.870728:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829759.870729:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829759.870730:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829759.870732:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829759.870733:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829759.870734:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829759.870736:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829759.870742:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870745:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829759.870746:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870747:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.870748:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829759.870751:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870755:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829759.870757:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870761:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829759.870763:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870764:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829759.870765:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870766:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829759.870767:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829759.870768:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829759.870770:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829759.870772:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829759.870774:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829759.870779:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376640:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829759.870781:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870783:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829759.870785:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327376640/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829759.870789:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829759.870790:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829759.870791:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829759.870792:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829759.870793:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829759.870794:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829759.870795:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829759.870796:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:0.0:1787829759.870797:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829759.870798:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829759.870799:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829759.870801:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829759.870802:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870803:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829759.870805:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829759.870806:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829759.870807:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829759.870808:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829760.380775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829760.380781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829760.380782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829760.380786:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829760.380789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829760.380791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829760.380793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829760.380795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829760.380797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829760.572755:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298580652, at 4298580672
00000100:00000040:3.0:1787829760.572756:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298580652, at 4298580672
00000100:00000010:3.0:1787829760.572778:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000010:2.0:1787829760.572781:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00010000:00000001:0.0:1787829760.700843:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829760.700850:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829760.700857:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829760.700859:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829760.700863:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829760.700868:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829760.700871:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829760.700876:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829760.700879:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829760.700882:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829760.700885:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829760.700889:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829760.700913:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829760.700915:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829760.700917:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829761.404785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829761.404790:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829761.404792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829761.404797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829761.404801:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829761.404805:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829761.404807:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829761.404809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829761.404811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829761.724783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829761.724789:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829761.724793:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829761.724794:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829761.724796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829761.724797:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829761.724860:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829761.724866:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829761.724871:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829762.428738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829762.428746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829762.428747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829762.428752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829762.428756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829762.428759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829762.428761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829762.428763:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829762.428766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829762.748778:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829762.748786:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829762.748791:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829762.748792:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829762.748796:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829762.748798:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829762.748826:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829762.748830:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829762.748834:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829763.452773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829763.452779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829763.452780:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829763.452785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829763.452788:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829763.452791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829763.452792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829763.452795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829763.452797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829763.772741:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829763.772747:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829763.772752:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829763.772753:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829763.772755:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829763.772756:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829763.772773:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829763.772775:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829763.772778:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829764.476744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829764.476750:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829764.476752:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829764.476755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829764.476758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829764.476760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.476762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829764.476764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.476766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796774:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829764.796782:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829764.796785:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829764.796789:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796802:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829764.796804:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796807:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796809:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796814:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829764.796817:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829764.796818:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829764.796820:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796822:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829764.796823:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796825:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796827:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829764.796830:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829764.796832:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829764.796904:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829764.796909:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829764.796912:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829764.988755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829764.988766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829764.988769:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829764.988773:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829764.988779:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829764.988786:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3912
00000400:00000200:3.0:1787829764.988796:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829764.988801:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829764.988803:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829764.988806:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.988808:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829764.988810:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.988812:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829764.988815:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829764.988816:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829764.988819:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829764.988822:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829764.988824:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829764.988826:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829764.988828:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.988829:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829764.988830:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.988831:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829764.988834:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829764.988836:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829764.988837:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829764.988840:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829764.988842:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829764.988844:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.988846:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829764.988848:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829764.988856:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000020:00000040:3.0:1787829764.988859:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829764.988869:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829764.988871:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.988873:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829764.988875:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829764.988879:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829764.988885:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829764.988890:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829764.988897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829764.988905:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829764.988908:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829764.988912:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829764.988932:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829764.988935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829764.988939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829764.988941:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829764.988943:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829764.988946:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829764.988949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3912
00000400:00000200:3.0:1787829764.988954:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829764.988958:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829764.988961:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829764.988965:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787829764.988966:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.988970:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829764.988971:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787829764.988972:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829764.988975:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787829764.988976:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829764.988979:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829764.988981:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000040:0.0:1787829764.988983:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787829764.988984:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829764.988987:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829764.988991:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829764.988996:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829764.989002:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1787829764.989002:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829764.989004:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989006:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829764.989009:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787829764.989009:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989011:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829764.989013:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1787829764.989015:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829764.989016:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829764.989020:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:0.0:1787829764.989024:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327376768:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829764.989024:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829764.989029:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829764.989031:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829764.989035:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787829764.989035:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829764.989037:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829764.989040:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.989041:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829764.989044:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:0.0:1787829764.989048:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000008:00100000:3.0:1787829764.989049:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829764.989053:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787829764.989054:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989059:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829764.989062:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829764.989067:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000010:3.0:1787829764.989067:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829764.989086:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:0.0:1787829764.989090:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
02000000:00000001:3.0:1787829764.989091:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829764.989095:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989097:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:0.0:1787829764.989099:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376768, portal 25
02000000:00000001:3.0:1787829764.989099:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829764.989102:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989103:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829764.989106:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
02000000:00000010:3.0:1787829764.989111:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000040:0.0:1787829764.989113:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829764.989118:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829764.989128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829764.989133:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829764.989140:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989142:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1787829764.989144:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:3.0:1787829764.989144:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829764.989149:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327376768, offset 0
00000100:00000040:3.0:1787829764.989152:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829764.989159:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989167:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829764.989170:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:3.0:1787829764.989170:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829764.989176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829764.989179:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829764.989181:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829764.989185:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1787829764.989185:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829764.989191:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829764.989192:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829764.989199:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1787829764.989200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3912
00000400:00000200:0.0:1787829764.989203:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829764.989209:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40715:40715:256:4294967295] 192.168.203.116@tcp LPNI seq info [40715:40715:8:4294967295]
00000400:00000200:3.0:1787829764.989213:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.989217:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829764.989223:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829764.989226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829764.989231:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787829764.989235:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829764.989241:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989241:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:0.0:1787829764.989246:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000040:3.0:1787829764.989248:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989253:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.0:1787829764.989254:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:3.0:1787829764.989260:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.0:1787829764.989261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000020:00000040:3.0:1787829764.989264:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:0.1:1787829764.989269:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:3.0:1787829764.989271:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829764.989275:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:0.1:1787829764.989275:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000020:00000001:3.0:1787829764.989278:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000100:0.0:1787829764.989283:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:3.0:1787829764.989284:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:0.0:1787829764.989287:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00000040:3.0:1787829764.989290:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989293:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829764.989294:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787829764.989297:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989301:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829764.989302:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000040:3.0:1787829764.989305:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829764.989308:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00080000:3.0:1787829764.989310:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000010:0.0:1787829764.989311:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:3.0:1787829764.989313:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829764.989316:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829764.989317:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787829764.989319:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1787829764.989323:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829764.989324:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989329:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829764.989333:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829764.989337:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.989341:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1787829764.989345:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989351:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989357:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989359:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:3.0:1787829764.989360:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829764.989364:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787829764.989365:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829764.989369:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829764.989372:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829764.989373:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298585772
02000000:00000001:3.0:1787829764.989376:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829764.989379:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc4e0.
00000100:00000001:3.0:1787829764.989379:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829764.989381:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829764.989385:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989387:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000010:3.0:1787829764.989392:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829764.989397:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989399:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298585772
00000100:00000010:0.0:1787829764.989404:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc7e0.
00000100:00100000:3.0:1787829764.989406:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829764.989408:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829764.989412:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829764.989417:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989422:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829764.989428:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764]  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829764.989433:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829764.989437:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829764.989443:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829764.989446:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787829764.989447:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829764.989448:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829764.989454:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829764.989457:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:3.0:1787829764.989460:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989461:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:3.0:1787829764.989463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1787829764.989468:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829764.989469:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829764.989472:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829764.989473:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829764.989477:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829764.989482:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3912
00000400:00000200:3.0:1787829764.989488:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829764.989493:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787829764.989494:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829764.989496:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829764.989501:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989506:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829764.989510:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989513:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829764.989516:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000001:3.0:1787829764.989517:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829764.989519:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829764.989521:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829764.989523:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829764.989524:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829764.989525:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829764.989527:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829764.989529:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787829764.989529:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829764.989531:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829764.989536:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829764.989537:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.989541:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1787829764.989542:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829764.989551:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:0.0:1787829764.989552:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829764.989555:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829764.989560:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829764.989565:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829764.989567:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787829764.989569:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829764.989575:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829764.989575:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829764.989579:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829764.989583:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989584:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829764.989587:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829764.989589:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829764.989593:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829764.989595:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829764.989599:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829764.989602:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829764.989603:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:3.0:1787829764.989604:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829764.989609:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000100:00000001:0.0:1787829764.989610:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989614:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787829764.989615:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989617:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989622:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00100000:3.0:1787829764.989622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829764.989624:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829764.989631:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787829764.989636:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00080000:0.0:1787829764.989640:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:3.0:1787829764.989641:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989644:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787829764.989648:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829764.989653:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829764.989675:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829764.989680:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3922)
00000100:00100000:0.0:1787829764.989696:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327376768:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829764.989701:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989703:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.989705:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327376768/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989710:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829764.989712:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829764.989714:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829764.989717:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829764.989718:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829764.989719:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829764.989721:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829764.989722:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:0.0:1787829764.989726:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829764.989727:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829764.989729:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:0.0:1787829764.989731:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829764.989732:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989733:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989738:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829764.989740:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989741:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829764.989743:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829764.989744:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829764.989746:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989751:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829764.989752:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.989754:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829764.989755:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829764.989756:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829764.989758:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376896:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829764.989760:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829764.989762:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829764.989763:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.989764:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829764.989767:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829764.989768:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829764.989771:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829764.989773:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829764.989776:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829764.989778:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327376896, portal 10
00000100:00000001:0.0:1787829764.989780:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829764.989781:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00000040:0.0:1787829764.989786:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989796:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829764.989798:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829764.989799:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829764.989801:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327376896, offset 0
00000400:00000200:0.0:1787829764.989804:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829764.989807:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.989810:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829764.989812:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829764.989814:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829764.989816:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829764.989818:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829764.989820:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40716:40716:256:4294967295] 192.168.203.116@tcp LPNI seq info [40716:40716:8:4294967295]
00000400:00000200:0.0:1787829764.989822:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.989827:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829764.989830:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829764.989832:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829764.989835:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829764.989837:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829764.989840:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829764.989842:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829764.989856:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829764.989857:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829764.989859:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829764.989862:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829764.989864:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829764.989866:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829764.989872:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829764.989875:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989881:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.989883:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989888:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989889:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829764.989892:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829764.989895:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829764.989897:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829764.989899:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298585099
00000100:00000001:0.0:1787829764.989901:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298585772
00000100:00000001:0.0:1787829764.989902:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298585099
00000100:00000001:0.0:1787829764.989904:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298585772
00000100:00000001:0.0:1787829764.989905:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829764.989906:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.989908:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829764.989910:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764]  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829764.989915:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829764.989916:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829764.989917:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829764.989919:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829764.989920:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829764.989922:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829764.989924:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829764.989929:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989933:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829764.989934:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989935:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.989936:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829764.989938:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989942:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829764.989944:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989949:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829764.989951:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989952:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.989953:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.989954:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829764.989956:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829764.989957:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829764.989959:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829764.989960:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829764.989963:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829764.989967:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327376896:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829764.989970:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829764.989972:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.989976:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829764.989977:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.989979:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829764.989980:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829764.989981:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829764.989983:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377024:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829764.989985:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829764.989986:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829764.989987:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.989989:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829764.989991:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829764.989992:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829764.989995:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829764.989997:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829764.989999:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829764.990001:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377024, portal 4
00000100:00000001:0.0:1787829764.990002:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829764.990003:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:0.0:1787829764.990007:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990011:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829764.990013:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829764.990014:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829764.990016:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327377024, offset 0
00000400:00000200:0.0:1787829764.990018:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829764.990020:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.990022:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829764.990024:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829764.990026:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829764.990028:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829764.990029:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829764.990031:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40717:40717:256:4294967295] 192.168.203.116@tcp LPNI seq info [40717:40717:8:4294967295]
00000400:00000200:0.0:1787829764.990034:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.990038:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829764.990041:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829764.990043:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829764.990046:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829764.990048:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829764.990050:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829764.990052:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829764.990054:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829764.990056:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829764.990058:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829764.990060:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829764.990062:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829764.990063:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829764.990065:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829764.990067:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990072:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.990090:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990095:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.990096:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829764.990099:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829764.990101:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829764.990103:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829764.990105:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298585106
00000100:00000001:0.0:1787829764.990106:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298585772
00000100:00000001:0.0:1787829764.990108:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298585106
00000100:00000001:0.0:1787829764.990109:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298585772
00000100:00000001:0.0:1787829764.990111:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829764.990112:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.990113:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829764.990116:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764]  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829764.990120:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829764.990121:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829764.990122:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829764.990124:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829764.990125:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829764.990126:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829764.990128:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829764.990134:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990141:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829764.990143:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990144:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.990145:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829764.990147:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990151:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829764.990153:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990157:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829764.990159:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990160:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.990160:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990162:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829764.990163:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829764.990164:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829764.990166:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829764.990168:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829764.990170:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829764.990175:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377024:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829764.990177:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829764.990179:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990183:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829764.990184:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.990186:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829764.990187:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829764.990188:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829764.990190:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377152:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829764.990192:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829764.990194:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829764.990195:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829764.990196:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829764.990198:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829764.990199:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829764.990201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829764.990203:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829764.990205:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829764.990207:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377152, portal 4
00000100:00000001:0.0:1787829764.990208:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829764.990209:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:0.0:1787829764.990213:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990217:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829764.990218:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829764.990220:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829764.990221:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327377152, offset 0
00000400:00000200:0.0:1787829764.990224:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829764.990226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.990228:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829764.990231:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829764.990232:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829764.990234:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829764.990236:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829764.990238:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40718:40718:256:4294967295] 192.168.203.116@tcp LPNI seq info [40718:40718:8:4294967295]
00000400:00000200:0.0:1787829764.990240:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829764.990244:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829764.990248:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829764.990250:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829764.990252:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829764.990254:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829764.990257:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829764.990259:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829764.990261:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829764.990263:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829764.990265:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829764.990267:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829764.990269:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829764.990270:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829764.990271:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829764.990274:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990278:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.990280:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990284:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.990286:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829764.990287:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829764.990289:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829764.990291:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829764.990293:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298585111
00000100:00000001:0.0:1787829764.990294:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298585772
00000100:00000001:0.0:1787829764.990296:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298585111
00000100:00000001:0.0:1787829764.990297:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298585772
00000100:00000001:0.0:1787829764.990298:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829764.990299:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.990300:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829764.990302:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764]  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829764.990308:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829764.990308:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829764.990310:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829764.990311:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829764.990312:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829764.990313:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829764.990315:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829764.990320:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990323:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829764.990324:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990325:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.990326:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829764.990328:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990332:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829764.990334:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990339:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829764.990341:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990342:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829764.990342:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990343:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829764.990345:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829764.990346:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829764.990348:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829764.990349:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829764.990352:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829764.990357:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377152:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829764.990359:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990360:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.990362:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327376896/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990366:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829764.990367:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829764.990368:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829764.990370:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829764.990371:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829764.990372:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829764.990373:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829764.990374:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829764.990375:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829764.990376:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829764.990377:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:0.0:1787829764.990379:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829764.990380:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990381:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.990383:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327377024/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990387:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829764.990388:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829764.990389:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829764.990390:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829764.990391:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829764.990392:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829764.990393:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829764.990393:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829764.990395:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829764.990396:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829764.990397:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829764.990398:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829764.990399:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990400:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829764.990402:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327377152/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829764.990406:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829764.990407:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829764.990407:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829764.990409:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829764.990409:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829764.990410:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829764.990411:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829764.990412:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829764.990413:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829764.990414:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829764.990414:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:0.0:1787829764.990417:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829764.990417:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990418:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829764.990421:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829764.990422:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829764.990423:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829764.990424:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829765.500803:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829765.500811:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829765.500813:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829765.500818:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829765.500822:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829765.500826:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829765.500828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829765.500831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829765.500834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829765.692794:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298585772, at 4298585792
00000100:00000040:2.0:1787829765.692799:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298585772, at 4298585792
00000100:00000010:3.0:1787829765.692818:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000100:00000010:2.0:1787829765.692829:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00010000:00000001:0.0:1787829765.820769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829765.820777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829765.820783:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829765.820784:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829765.820787:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829765.820788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829765.820826:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829765.820830:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829765.820833:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829766.524775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829766.524783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829766.524785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829766.524789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829766.524792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829766.524796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829766.524798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829766.524801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829766.524803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829766.844760:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829766.844766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829766.844771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829766.844772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829766.844775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829766.844777:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829766.844798:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829766.844801:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829766.844804:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829767.548770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829767.548774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829767.548776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829767.548779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829767.548782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829767.548785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829767.548787:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829767.548789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829767.548790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829767.868732:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829767.868739:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829767.868745:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829767.868746:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829767.868748:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829767.868749:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829767.868794:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829767.868797:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829767.868800:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829768.572751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829768.572759:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829768.572762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829768.572767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829768.572771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829768.572776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829768.572779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829768.572782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829768.572785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829768.892760:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829768.892767:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829768.892774:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829768.892776:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829768.892779:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829768.892781:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829768.892818:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829768.892822:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829768.892825:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829769.596771:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829769.596777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829769.596779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829769.596784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829769.596786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829769.596790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829769.596792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829769.596794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829769.596796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829769.916746:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829769.916754:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829769.916761:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829769.916763:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829769.916767:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829769.916768:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829769.916795:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829769.916797:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829769.916799:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829770.108793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829770.108804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829770.108807:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829770.108810:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829770.108815:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829770.108823:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3917
00000400:00000200:3.0:1787829770.108832:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829770.108843:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829770.108847:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829770.108851:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.108854:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829770.108856:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.108859:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829770.108863:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829770.108865:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829770.108867:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829770.108871:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829770.108873:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829770.108876:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829770.108878:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.108881:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829770.108883:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.108885:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829770.108888:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829770.108898:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829770.108904:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829770.108907:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829770.108911:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829770.108914:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.108916:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829770.108918:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829770.108927:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829770.108930:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829770.108934:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829770.108936:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.108939:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829770.108942:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829770.108946:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829770.108954:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0800.
00000100:00000001:3.0:1787829770.108960:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829770.108969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829770.108979:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829770.108983:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829770.108988:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829770.109010:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829770.109014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829770.109017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829770.109018:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829770.109020:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829770.109023:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829770.109025:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3917
00000400:00000200:3.0:1787829770.109028:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829770.109031:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829770.109034:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829770.109036:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109055:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829770.109058:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109061:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829770.109064:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829770.109065:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829770.109067:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829770.109070:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829770.109072:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829770.109074:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1787829770.109075:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829770.109081:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829770.109082:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829770.109084:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829770.109087:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787829770.109088:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109092:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829770.109095:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:0.0:1787829770.109099:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829770.109100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109106:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829770.109111:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829770.109118:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829770.109120:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787829770.109122:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109124:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.109126:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829770.109129:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1787829770.109132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829770.109133:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1787829770.109139:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.109141:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.109144:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829770.109147:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000100:00100000:0.0:1787829770.109149:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327377280:192.168.203.116@tcp:250:kworker.0
00000020:00000040:3.0:1787829770.109153:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:0.0:1787829770.109155:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829770.109160:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787829770.109160:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829770.109163:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.109164:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.109170:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787829770.109170:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829770.109175:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1787829770.109178:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:3.0:1787829770.109179:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.109183:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829770.109189:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000400:00000200:0.0:1787829770.109190:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829770.109197:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829770.109199:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00100000:3.0:1787829770.109206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829770.109209:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829770.109218:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377280, portal 25
00000100:00000001:3.0:1787829770.109219:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829770.109223:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109224:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829770.109229:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:3.0:1787829770.109230:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829770.109240:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.109268:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787829770.109269:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829770.109275:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829770.109276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787829770.109282:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00080000:3.0:1787829770.109282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829770.109285:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829770.109289:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1787829770.109290:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327377280, offset 0
00000100:00000001:3.0:1787829770.109292:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829770.109296:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3917
00000400:00000200:3.0:1787829770.109302:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829770.109308:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829770.109312:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:3.0:1787829770.109313:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829770.109320:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829770.109323:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829770.109325:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829770.109331:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:3.0:1787829770.109333:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109338:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829770.109339:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:3.0:1787829770.109344:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829770.109345:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000040:3.0:1787829770.109348:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:0.0:1787829770.109353:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000001:3.0:1787829770.109354:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829770.109359:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829770.109360:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000001:3.0:1787829770.109364:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829770.109371:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40719:40719:256:4294967295] 192.168.203.116@tcp LPNI seq info [40719:40719:8:4294967295]
00000020:00000001:3.0:1787829770.109371:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829770.109379:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829770.109384:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829770.109385:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829770.109390:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109393:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829770.109399:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829770.109404:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1787829770.109407:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109413:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829770.109416:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829770.109418:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829770.109423:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:0.0:1787829770.109424:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
02000000:00000001:3.0:1787829770.109429:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.109434:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829770.109435:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00100000:3.0:1787829770.109438:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.0:1787829770.109441:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000008:00000001:3.0:1787829770.109443:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.109448:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829770.109453:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:3.0:1787829770.109454:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000020:00000040:3.0:1787829770.109461:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.1:1787829770.109461:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
02000000:00000001:3.0:1787829770.109467:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829770.109470:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829770.109472:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109474:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:0.0:1787829770.109476:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000001:3.0:1787829770.109477:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829770.109481:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829770.109487:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829770.109491:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829770.109494:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:3.0:1787829770.109498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829770.109502:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829770.109509:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829770.109514:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:3.0:1787829770.109514:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829770.109519:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109521:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787829770.109527:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829770.109528:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829770.109561:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829770.109564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829770.109565:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1787829770.109571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000040:0.0:1787829770.109572:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829770.109577:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829770.109581:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829770.109583:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109587:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787829770.109587:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829770.109594:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829770.109594:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3917
00000400:00000200:0.0:1787829770.109605:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829770.109605:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829770.109609:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298590892
00000400:00000200:3.0:1787829770.109613:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000010:0.0:1787829770.109616:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc300.
00000100:00080000:3.0:1787829770.109617:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829770.109623:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787829770.109623:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109629:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829770.109632:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109635:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829770.109637:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298590892
00000020:00000001:3.0:1787829770.109638:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000010:0.0:1787829770.109642:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf00.
00000020:00000040:3.0:1787829770.109643:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829770.109646:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000001:3.0:1787829770.109647:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829770.109650:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1787829770.109652:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829770.109653:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829770.109679:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829770.109683:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829770.109687:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109690:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829770.109693:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829770.109698:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1787829770.109699:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:0.0:1787829770.109705:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829770/real 1787829770]  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787829770.109706:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829770.109716:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829770.109724:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829770.109729:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000001:0.0:1787829770.109730:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829770.109734:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829770.109736:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
02000000:00000001:3.0:1787829770.109736:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829770.109739:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829770.109743:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
02000000:00000001:3.0:1787829770.109744:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109745:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787829770.109747:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829770.109751:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829770.109754:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.109755:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829770.109757:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787829770.109759:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000020:00000040:3.0:1787829770.109764:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829770.109769:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:0.0:1787829770.109773:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829770.109773:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829770.109776:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829770.109779:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829770.109782:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109785:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000010:3.0:1787829770.109787:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:0.0:1787829770.109788:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.109793:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787829770.109793:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109794:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787829770.109806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829770.109807:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829770.109827:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829770.109831:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109833:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787829770.109837:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829770.109844:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829770.109846:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829770.109851:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3927)
00000100:00000001:0.0:1787829770.109859:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829770.109863:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.109866:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.109868:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.109871:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829770.109873:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829770.109877:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829770.109882:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829770.109884:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829770.109889:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829770.109896:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327377280:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829770.109899:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.109901:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.109903:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327377280/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.109907:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829770.109909:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829770.109911:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829770.109914:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829770.109915:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829770.109916:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829770.109918:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829770.109920:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0800.
02000000:00000001:0.0:1787829770.109922:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829770.109924:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829770.109925:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829770.109927:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829770.109928:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.109930:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.109935:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829770.109937:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.109938:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829770.109939:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829770.109940:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829770.109943:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.109948:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829770.109949:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.109950:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829770.109952:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829770.109953:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829770.109955:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377408:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829770.109958:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829770.109959:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829770.109961:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.109962:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829770.109964:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829770.109965:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829770.109968:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829770.109970:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829770.109973:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829770.109975:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377408, portal 10
00000100:00000001:0.0:1787829770.109977:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829770.109978:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:0.0:1787829770.109983:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.109987:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829770.109989:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829770.109990:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829770.109992:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327377408, offset 0
00000400:00000200:0.0:1787829770.109995:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829770.109998:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829770.110000:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829770.110002:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829770.110004:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829770.110006:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829770.110008:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829770.110010:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40720:40720:256:4294967295] 192.168.203.116@tcp LPNI seq info [40720:40720:8:4294967295]
00000400:00000200:0.0:1787829770.110013:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829770.110017:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829770.110020:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829770.110022:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829770.110025:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829770.110028:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829770.110031:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829770.110033:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829770.110036:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829770.110050:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829770.110053:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829770.110056:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829770.110058:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829770.110060:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829770.110061:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829770.110064:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110070:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.110072:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110078:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110080:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829770.110083:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829770.110087:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829770.110091:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829770.110093:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298590219
00000100:00000001:0.0:1787829770.110096:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298590892
00000100:00000001:0.0:1787829770.110098:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298590219
00000100:00000001:0.0:1787829770.110101:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298590892
00000100:00000001:0.0:1787829770.110103:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829770.110105:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110107:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829770.110111:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829770/real 1787829770]  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829770.110120:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829770.110122:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829770.110123:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829770.110125:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829770.110126:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829770.110128:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829770.110130:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829770.110136:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110140:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829770.110141:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110142:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.110143:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829770.110145:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110149:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829770.110152:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110156:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829770.110158:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110159:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.110160:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110161:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829770.110162:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829770.110164:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829770.110166:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829770.110167:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829770.110169:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829770.110174:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377408:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829770.110177:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829770.110179:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110183:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829770.110184:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.110186:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829770.110187:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829770.110188:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829770.110190:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377536:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829770.110192:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829770.110194:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829770.110196:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.110198:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829770.110201:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829770.110203:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829770.110207:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829770.110209:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829770.110212:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829770.110215:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377536, portal 4
00000100:00000001:0.0:1787829770.110218:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829770.110219:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:0.0:1787829770.110225:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110232:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829770.110235:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829770.110238:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829770.110241:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327377536, offset 0
00000400:00000200:0.0:1787829770.110245:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829770.110249:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829770.110252:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829770.110255:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829770.110258:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829770.110260:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829770.110263:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829770.110266:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40721:40721:256:4294967295] 192.168.203.116@tcp LPNI seq info [40721:40721:8:4294967295]
00000400:00000200:0.0:1787829770.110270:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829770.110276:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829770.110282:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829770.110286:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829770.110290:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829770.110293:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829770.110298:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829770.110301:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829770.110305:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829770.110306:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829770.110309:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829770.110311:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829770.110313:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829770.110315:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829770.110316:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829770.110319:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110325:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.110327:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110331:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110332:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829770.110335:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829770.110337:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829770.110339:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829770.110341:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298590226
00000100:00000001:0.0:1787829770.110342:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298590892
00000100:00000001:0.0:1787829770.110344:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298590226
00000100:00000001:0.0:1787829770.110346:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298590892
00000100:00000001:0.0:1787829770.110347:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829770.110349:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110351:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829770.110355:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829770/real 1787829770]  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829770.110363:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829770.110365:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829770.110366:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829770.110369:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829770.110370:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829770.110372:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829770.110376:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829770.110385:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110392:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829770.110393:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110396:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.110397:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829770.110400:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110405:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829770.110408:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110412:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829770.110413:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110414:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.110415:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110417:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829770.110418:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829770.110420:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829770.110422:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829770.110424:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829770.110426:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829770.110431:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377536:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829770.110434:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829770.110437:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110443:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829770.110445:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.110447:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829770.110449:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829770.110451:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829770.110454:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377664:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829770.110458:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829770.110459:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829770.110461:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829770.110463:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829770.110467:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829770.110469:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829770.110473:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829770.110476:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829770.110479:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829770.110483:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377664, portal 4
00000100:00000001:0.0:1787829770.110484:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829770.110485:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:0.0:1787829770.110490:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110495:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829770.110496:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829770.110498:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829770.110500:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327377664, offset 0
00000400:00000200:0.0:1787829770.110502:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829770.110505:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829770.110507:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829770.110509:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829770.110511:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829770.110513:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829770.110514:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829770.110516:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40722:40722:256:4294967295] 192.168.203.116@tcp LPNI seq info [40722:40722:8:4294967295]
00000400:00000200:0.0:1787829770.110519:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829770.110524:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829770.110526:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829770.110528:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829770.110531:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829770.110533:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787829770.110536:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829770.110537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1787829770.110540:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829770.110541:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829770.110544:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829770.110546:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829770.110548:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829770.110549:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829770.110551:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829770.110553:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110558:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.110560:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110569:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110570:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829770.110572:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829770.110574:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829770.110576:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829770.110577:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298590231
00000100:00000001:0.0:1787829770.110579:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298590892
00000100:00000001:0.0:1787829770.110581:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298590231
00000100:00000001:0.0:1787829770.110582:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298590892
00000100:00000001:0.0:1787829770.110584:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829770.110585:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110586:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829770.110588:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829770/real 1787829770]  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829770.110593:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829770.110594:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829770.110595:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829770.110596:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829770.110597:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829770.110598:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829770.110600:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829770.110605:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110609:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829770.110610:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110611:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.110612:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829770.110614:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110620:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829770.110622:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110627:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829770.110628:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110629:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829770.110630:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110631:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829770.110632:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829770.110633:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829770.110635:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829770.110637:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829770.110639:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829770.110644:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377664:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829770.110646:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110647:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.110649:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327377408/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110653:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829770.110671:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829770.110672:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829770.110675:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829770.110676:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829770.110677:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829770.110678:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829770.110680:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:0.0:1787829770.110682:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829770.110683:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829770.110684:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829770.110686:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829770.110687:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110688:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.110690:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327377536/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110695:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829770.110696:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829770.110697:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829770.110698:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829770.110699:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829770.110700:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829770.110701:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829770.110702:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:0.0:1787829770.110703:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829770.110704:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829770.110705:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:0.0:1787829770.110706:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829770.110706:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110707:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829770.110709:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327377664/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829770.110713:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829770.110714:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829770.110714:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829770.110716:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829770.110716:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829770.110717:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829770.110718:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829770.110719:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:0.0:1787829770.110720:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829770.110721:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829770.110721:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:0.0:1787829770.110724:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829770.110725:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110726:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.110729:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829770.110730:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.110731:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829770.110732:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829770.620788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829770.620798:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829770.620801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829770.620809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829770.620814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829770.620819:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.620823:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829770.620827:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829770.620832:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829770.812717:0:16224:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298590892, at 4298590912
00000100:00000040:2.0:1787829770.812719:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298590892, at 4298590912
00000100:00000010:3.0:1787829770.812733:0:16224:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00000010:2.0:1787829770.812737:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00010000:00000001:0.0:1787829770.940790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829770.940797:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829770.940804:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829770.940807:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829770.940812:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829770.940815:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829770.940818:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829770.940820:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829770.940823:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829770.940825:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829770.940827:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829770.940829:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829770.940884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829770.940888:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829770.940891:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829771.644759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829771.644764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829771.644766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829771.644770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829771.644773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829771.644776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829771.644778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829771.644780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829771.644782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829771.964835:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829771.964841:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829771.964848:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829771.964849:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829771.964858:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829771.964860:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829771.964958:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829771.964978:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829771.964984:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829772.668740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829772.668746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829772.668749:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829772.668755:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829772.668758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829772.668762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829772.668764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829772.668767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829772.668770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829772.988815:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829772.988824:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829772.988831:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829772.988832:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829772.988835:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829772.988837:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829772.988866:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829772.988870:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829772.988873:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829773.692801:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829773.692815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829773.692818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829773.692825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829773.692829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829773.692834:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829773.692837:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829773.692840:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829773.692844:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829774.013071:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829774.013080:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829774.013088:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829774.013090:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829774.013095:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829774.013096:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829774.013166:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829774.013172:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829774.013177:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829774.716739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829774.716746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829774.716748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829774.716754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829774.716758:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829774.716761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829774.716764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829774.716766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829774.716769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036746:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829775.036755:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829775.036757:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829775.036760:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036765:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829775.036767:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036769:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036772:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036775:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829775.036777:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829775.036778:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829775.036779:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036781:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829775.036782:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036784:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036785:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829775.036787:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829775.036788:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829775.036825:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829775.036830:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829775.036833:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829775.228758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829775.228767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829775.228769:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829775.228773:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829775.228777:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829775.228784:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3922
00000400:00000200:1.0:1787829775.228792:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829775.228798:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829775.228801:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829775.228804:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228806:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829775.228809:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228812:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829775.228816:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829775.228817:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829775.228820:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829775.228823:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829775.228825:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829775.228828:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829775.228831:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228833:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829775.228835:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228837:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829775.228840:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829775.228844:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228846:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829775.228849:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829775.228852:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829775.228855:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.228857:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829775.228859:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829775.228867:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829775.228869:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829775.228872:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829775.228874:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.228876:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829775.228879:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829775.228883:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829775.228889:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829775.228894:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829775.228902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829775.228911:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829775.228915:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829775.228919:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829775.228936:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829775.228939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829775.228942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829775.228944:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829775.228945:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829775.228948:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829775.228950:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3922
00000400:00000200:1.0:1787829775.228953:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829775.228956:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829775.228958:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829775.228960:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228962:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829775.228964:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829775.228965:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829775.228965:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829775.228968:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829775.228969:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1787829775.228970:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829775.228972:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787829775.228973:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829775.228976:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829775.228979:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829775.228980:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829775.228981:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829775.228984:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228987:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829775.228990:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.228993:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829775.228996:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829775.228999:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829775.229001:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.229012:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829775.229015:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787829775.229016:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:0.0:1787829775.229019:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.229023:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829775.229026:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327377792:192.168.203.116@tcp:250:kworker.0
02000000:00000001:1.0:1787829775.229026:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829775.229031:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829775.229031:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829775.229034:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.229035:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829775.229036:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829775.229038:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787829775.229040:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829775.229042:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:1.0:1787829775.229044:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.229046:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829775.229049:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829775.229050:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829775.229051:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829775.229052:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829775.229053:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
00000100:00000001:1.0:1787829775.229054:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829775.229056:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829775.229057:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829775.229058:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829775.229063:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000400:00000200:0.0:1787829775.229064:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829775.229067:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829775.229069:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00100000:1.0:1787829775.229073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829775.229075:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829775.229081:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:0.0:1787829775.229082:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377792, portal 25
00000100:00000001:1.0:1787829775.229084:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229086:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829775.229088:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:1.0:1787829775.229091:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829775.229095:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229104:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829775.229106:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829775.229109:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829775.229114:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327377792, offset 0
00000100:00000001:1.0:1787829775.229116:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829775.229121:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829775.229121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829775.229127:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829775.229127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829775.229131:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829775.229133:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829775.229134:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829775.229137:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829775.229140:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1787829775.229142:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3922
00000400:00000200:0.0:1787829775.229149:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829775.229153:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829775.229155:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829775.229158:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40723:40723:256:4294967295] 192.168.203.116@tcp LPNI seq info [40723:40723:8:4294967295]
00000400:00000200:0.0:1787829775.229161:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829775.229165:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.229173:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:1.0:1787829775.229173:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829775.229175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829775.229181:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829775.229182:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829775.229183:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829775.229187:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:0.0:1787829775.229188:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829775.229189:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829775.229192:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829775.229193:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829775.229194:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829775.229197:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829775.229198:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829775.229200:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829775.229200:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829775.229205:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829775.229206:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1787829775.229207:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829775.229211:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.229213:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829775.229215:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:1.0:1787829775.229216:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787829775.229219:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829775.229224:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829775.229225:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000010:0.0:1787829775.229227:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829775.229228:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829775.229230:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829775.229233:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:1.0:1787829775.229233:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829775.229236:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.229239:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829775.229240:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000008:00100000:1.0:1787829775.229242:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829775.229244:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000008:00000001:1.0:1787829775.229245:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829775.229248:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:1.0:1787829775.229249:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229255:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:1.0:1787829775.229255:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829775.229259:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000200:0.0:1787829775.229260:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829775.229261:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829775.229263:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.229266:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829775.229268:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829775.229270:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229273:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:1.0:1787829775.229276:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00000040:0.0:1787829775.229277:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829775.229282:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229284:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229286:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829775.229291:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829775.229291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829775.229296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829775.229298:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298596012
00000100:00000010:0.0:1787829775.229302:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc720.
00000100:00000001:0.0:1787829775.229308:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829775.229310:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829775.229314:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829775.229318:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229319:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298596012
00000100:00000010:0.0:1787829775.229324:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc480.
00000100:00000001:0.0:1787829775.229329:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829775.229331:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229332:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787829775.229333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829775.229336:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829775.229337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829775.229339:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829775.229340:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829775.229342:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:0.0:1787829775.229345:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829775/real 1787829775]  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829775.229346:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829775.229351:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3922
00000400:00000001:0.0:1787829775.229358:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:1.0:1787829775.229358:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000001:0.0:1787829775.229361:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:1.0:1787829775.229363:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000010:0.0:1787829775.229364:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00080000:1.0:1787829775.229365:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829775.229369:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000100:00000040:1.0:1787829775.229370:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829775.229371:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829775.229373:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829775.229375:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:1.0:1787829775.229376:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.229378:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829775.229380:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829775.229381:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829775.229383:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829775.229385:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829775.229387:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829775.229390:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829775.229393:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829775.229395:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229400:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829775.229402:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829775.229403:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829775.229404:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829775.229406:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829775.229407:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829775.229409:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829775.229412:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:0.0:1787829775.229413:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829775.229414:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829775.229419:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829775.229424:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829775.229427:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829775.229429:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829775.229430:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829775.229433:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829775.229435:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829775.229435:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829775.229438:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829775.229444:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:1.0:1787829775.229447:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829775.229449:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229452:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:1.0:1787829775.229452:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829775.229456:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:1.0:1787829775.229456:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829775.229459:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829775.229460:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829775.229461:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:1.0:1787829775.229462:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229464:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829775.229467:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787829775.229468:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00080000:0.0:1787829775.229472:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
02000000:00000001:1.0:1787829775.229472:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829775.229474:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229476:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000010:1.0:1787829775.229480:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000040:0.0:1787829775.229483:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829775.229489:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829775.229499:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829775.229500:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327377792:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829775.229506:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829775.229508:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829775.229513:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229520:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829775.229522:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229524:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.229529:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327377792/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829775.229530:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3933)
00000100:00000001:0.0:1787829775.229538:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829775.229541:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829775.229544:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829775.229555:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829775.229557:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829775.229558:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829775.229560:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829775.229563:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829775.229567:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829775.229569:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829775.229570:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829775.229573:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829775.229574:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.229575:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.229581:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829775.229582:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229583:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829775.229584:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829775.229585:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829775.229588:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229593:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829775.229593:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.229595:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829775.229596:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829775.229598:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829775.229599:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377920:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829775.229601:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829775.229603:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829775.229604:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.229605:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829775.229607:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829775.229609:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829775.229612:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829775.229614:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829775.229618:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829775.229621:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327377920, portal 10
00000100:00000001:0.0:1787829775.229622:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829775.229623:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829775.229630:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229636:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829775.229638:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829775.229641:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829775.229643:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327377920, offset 0
00000400:00000200:0.0:1787829775.229647:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829775.229650:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.229654:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829775.229675:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829775.229677:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829775.229678:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829775.229680:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829775.229682:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40724:40724:256:4294967295] 192.168.203.116@tcp LPNI seq info [40724:40724:8:4294967295]
00000400:00000200:0.0:1787829775.229684:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.229689:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829775.229692:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829775.229694:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829775.229697:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829775.229698:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829775.229701:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829775.229703:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829775.229705:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829775.229706:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829775.229708:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829775.229710:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829775.229712:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829775.229713:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829775.229715:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829775.229718:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229725:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.229728:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229735:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229736:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829775.229739:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829775.229742:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829775.229745:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829775.229748:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298595339
00000100:00000001:0.0:1787829775.229750:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298596012
00000100:00000001:0.0:1787829775.229752:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298595339
00000100:00000001:0.0:1787829775.229755:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298596012
00000100:00000001:0.0:1787829775.229757:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829775.229759:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229760:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829775.229763:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829775/real 1787829775]  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829775.229769:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829775.229770:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829775.229771:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829775.229773:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829775.229774:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829775.229775:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829775.229778:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829775.229782:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229786:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829775.229787:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.229788:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829775.229788:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829775.229790:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229795:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829775.229798:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229812:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829775.229814:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.229816:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829775.229817:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.229818:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829775.229820:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829775.229823:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829775.229826:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829775.229829:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829775.229832:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829775.229839:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327377920:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829775.229841:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829775.229848:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229860:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829775.229862:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.229864:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829775.229865:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829775.229868:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829775.229871:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378048:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829775.229874:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829775.229876:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829775.229877:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.229879:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829775.229882:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829775.229884:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829775.229887:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829775.229889:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829775.229891:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829775.229894:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378048, portal 4
00000100:00000001:0.0:1787829775.229895:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829775.229897:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829775.229902:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229908:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829775.229910:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829775.229912:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829775.229914:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327378048, offset 0
00000400:00000200:0.0:1787829775.229918:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829775.229921:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.229924:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829775.229927:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829775.229930:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829775.229932:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829775.229934:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829775.229937:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40725:40725:256:4294967295] 192.168.203.116@tcp LPNI seq info [40725:40725:8:4294967295]
00000400:00000200:0.0:1787829775.229939:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.229944:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829775.229946:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829775.229949:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829775.229951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829775.229953:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829775.229955:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829775.229957:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829775.229959:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829775.229960:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829775.229962:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829775.229964:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829775.229966:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829775.229967:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829775.229969:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829775.229971:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229975:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.229977:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.229981:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229982:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829775.229983:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829775.229985:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829775.229987:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829775.229988:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298595346
00000100:00000001:0.0:1787829775.229990:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298596012
00000100:00000001:0.0:1787829775.229991:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298595346
00000100:00000001:0.0:1787829775.229992:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298596012
00000100:00000001:0.0:1787829775.229993:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829775.229995:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.229996:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829775.229998:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829775/real 1787829775]  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829775.230014:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829775.230015:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829775.230016:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829775.230018:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829775.230019:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829775.230020:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829775.230023:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829775.230028:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230031:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829775.230032:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230033:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829775.230034:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829775.230036:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230040:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829775.230042:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230046:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829775.230047:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230048:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829775.230049:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230050:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829775.230050:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829775.230052:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829775.230054:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829775.230055:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829775.230057:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829775.230062:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378048:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829775.230064:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829775.230065:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230069:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829775.230070:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.230071:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829775.230072:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829775.230073:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829775.230074:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378176:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829775.230076:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829775.230077:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829775.230078:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829775.230079:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829775.230081:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829775.230082:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829775.230084:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829775.230085:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829775.230087:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829775.230089:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378176, portal 4
00000100:00000001:0.0:1787829775.230090:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829775.230091:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:0.0:1787829775.230094:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230098:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829775.230099:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829775.230101:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829775.230102:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327378176, offset 0
00000400:00000200:0.0:1787829775.230104:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829775.230106:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.230108:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829775.230110:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829775.230112:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829775.230113:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829775.230115:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829775.230117:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40726:40726:256:4294967295] 192.168.203.116@tcp LPNI seq info [40726:40726:8:4294967295]
00000400:00000200:0.0:1787829775.230119:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829775.230123:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829775.230126:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829775.230127:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829775.230130:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829775.230131:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829775.230133:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829775.230135:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829775.230137:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829775.230138:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829775.230140:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829775.230142:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829775.230143:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829775.230144:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829775.230146:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829775.230148:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230152:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.230154:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230157:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.230158:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829775.230160:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829775.230162:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829775.230163:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829775.230165:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298595351
00000100:00000001:0.0:1787829775.230166:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298596012
00000100:00000001:0.0:1787829775.230167:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298595351
00000100:00000001:0.0:1787829775.230168:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298596012
00000100:00000001:0.0:1787829775.230169:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829775.230170:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.230171:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829775.230173:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829775/real 1787829775]  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829775.230177:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829775.230178:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829775.230179:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829775.230180:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829775.230181:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829775.230182:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829775.230184:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829775.230188:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230192:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829775.230193:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230194:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829775.230194:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829775.230196:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230200:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829775.230202:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230206:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829775.230208:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230209:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829775.230209:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230210:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829775.230211:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829775.230212:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829775.230214:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829775.230215:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829775.230217:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829775.230221:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378176:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829775.230223:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230224:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.230225:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327377920/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230228:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829775.230229:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829775.230230:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829775.230232:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829775.230232:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829775.230233:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829775.230234:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829775.230235:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829775.230237:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829775.230237:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829775.230238:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829775.230240:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829775.230241:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230242:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.230244:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327378048/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230247:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829775.230248:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829775.230249:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829775.230250:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829775.230251:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829775.230251:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829775.230252:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829775.230253:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829775.230254:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829775.230254:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829775.230255:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829775.230256:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829775.230257:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230258:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829775.230259:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327378176/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829775.230262:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829775.230263:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829775.230264:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829775.230264:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829775.230265:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829775.230266:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829775.230266:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829775.230267:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829775.230268:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829775.230269:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829775.230269:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829775.230271:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829775.230271:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230272:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.230274:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829775.230275:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.230276:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829775.230277:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829775.740741:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829775.740746:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829775.740748:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829775.740753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829775.740757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829775.740760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.740762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829775.740764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829775.740771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829775.932748:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298596012, at 4298596032
00000100:00000040:2.0:1787829775.932750:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298596012, at 4298596032
00000100:00000010:3.0:1787829775.932760:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00000100:00000010:2.0:1787829775.932763:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00010000:00000001:0.0:1787829776.060761:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829776.060765:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829776.060769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829776.060770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829776.060773:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829776.060774:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829776.060796:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829776.060799:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829776.060802:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829776.764758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829776.764763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829776.764765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829776.764770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829776.764773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829776.764777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829776.764779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829776.764782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829776.764785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829777.084759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829777.084764:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829777.084768:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829777.084769:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829777.084771:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829777.084772:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829777.084828:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829777.084832:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829777.084836:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829777.788757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829777.788763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829777.788765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829777.788769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829777.788773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829777.788776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829777.788778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829777.788781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829777.788783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829778.108757:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829778.108762:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829778.108766:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829778.108767:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829778.108769:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829778.108770:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829778.108803:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829778.108807:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829778.108811:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829778.812765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829778.812771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829778.812773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829778.812778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829778.812782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829778.812785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829778.812788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829778.812790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829778.812793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829779.132777:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829779.132782:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829779.132786:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829779.132787:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829779.132790:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829779.132792:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829779.132811:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829779.132814:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829779.132817:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829779.836747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829779.836754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829779.836757:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829779.836762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829779.836766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829779.836769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829779.836772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829779.836774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829779.836777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829780.156810:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829780.156817:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829780.156824:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829780.156826:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829780.156830:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829780.156832:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829780.156895:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829780.156899:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829780.156902:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829780.348799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829780.348813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829780.348815:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829780.348819:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829780.348823:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829780.348830:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3928
00000400:00000200:1.0:1787829780.348839:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829780.348845:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829780.348849:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829780.348852:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.348855:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.348857:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.348858:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829780.348862:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829780.348863:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829780.348866:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829780.348869:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829780.348871:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829780.348872:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829780.348874:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.348876:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.348877:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.348878:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829780.348880:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829780.348883:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829780.348884:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829780.348886:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829780.348893:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829780.348895:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.348901:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829780.348904:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829780.348917:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829780.348920:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829780.348925:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829780.348927:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.348930:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829780.348933:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829780.348938:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829780.348949:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:1.0:1787829780.348956:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829780.348979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829780.348990:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829780.348995:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829780.349000:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829780.349021:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829780.349024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829780.349027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829780.349028:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829780.349030:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829780.349033:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829780.349035:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3928
00000400:00000200:1.0:1787829780.349039:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829780.349043:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829780.349045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829780.349049:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349051:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.349055:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349057:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829780.349059:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829780.349060:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829780.349064:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787829780.349064:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829780.349070:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829780.349072:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787829780.349075:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829780.349078:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787829780.349080:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829780.349085:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349087:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.349092:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829780.349096:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829780.349096:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829780.349101:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829780.349105:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349113:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829780.349120:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829780.349127:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829780.349129:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829780.349130:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829780.349141:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.349142:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787829780.349144:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829780.349146:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829780.349152:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787829780.349152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1787829780.349165:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327378304:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829780.349169:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:1.0:1787829780.349170:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.349173:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829780.349174:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.349178:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829780.349180:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829780.349185:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829780.349186:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787829780.349189:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829780.349191:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.349193:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829780.349194:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829780.349196:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829780.349197:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829780.349207:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829780.349209:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829780.349215:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829780.349221:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:1.0:1787829780.349226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829780.349230:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000001:1.0:1787829780.349240:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829780.349243:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829780.349246:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000040:1.0:1787829780.349251:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829780.349255:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378304, portal 25
00000100:00000001:0.0:1787829780.349265:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829780.349269:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000001:1.0:1787829780.349277:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829780.349284:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829780.349284:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829780.349294:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829780.349296:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829780.349299:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829780.349308:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829780.349314:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3928
00000100:00000001:0.0:1787829780.349319:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787829780.349322:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829780.349328:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829780.349332:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829780.349338:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327378304, offset 0
00000400:00000200:0.0:1787829780.349344:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829780.349349:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829780.349352:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829780.349358:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829780.349358:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349364:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.349368:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829780.349373:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829780.349380:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829780.349382:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1787829780.349385:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829780.349388:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:0.0:1787829780.349389:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:1.0:1787829780.349394:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829780.349397:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:1.0:1787829780.349400:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829780.349404:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000001:1.0:1787829780.349406:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829780.349412:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829780.349413:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40727:40727:256:4294967295] 192.168.203.116@tcp LPNI seq info [40727:40727:8:4294967295]
00000400:00000200:0.0:1787829780.349423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829780.349429:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349432:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.349441:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829780.349443:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787829780.349452:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829780.349458:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.0:1787829780.349460:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829780.349465:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000010:0.0:1787829780.349470:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829780.349483:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829780.349486:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:1.0:1787829780.349490:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829780.349495:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829780.349500:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.349503:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829780.349504:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000008:00100000:1.0:1787829780.349506:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829780.349514:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.349516:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829780.349521:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000800:00000200:0.1:1787829780.349522:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:1.0:1787829780.349526:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.1:1787829780.349529:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829780.349535:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.349542:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000001:1.0:1787829780.349543:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829780.349546:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.349550:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829780.349551:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829780.349553:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:1.0:1787829780.349553:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829780.349557:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829780.349562:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829780.349565:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00100000:1.0:1787829780.349570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829780.349574:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00000001:1.0:1787829780.349579:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1787829780.349581:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:1.0:1787829780.349584:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.349589:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787829780.349594:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829780.349601:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829780.349626:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829780.349633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829780.349635:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.349641:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829780.349643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829780.349647:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829780.349651:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829780.349688:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.349691:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829780.349697:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.349697:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829780.349703:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3928
00000400:00000200:0.0:1787829780.349706:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829780.349713:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298601133
00000400:00000200:1.0:1787829780.349716:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:0.0:1787829780.349721:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc060.
00000400:00000200:1.0:1787829780.349724:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829780.349729:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829780.349730:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787829780.349736:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829780.349739:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298601133
00000100:00000001:1.0:1787829780.349740:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.349747:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:0.0:1787829780.349749:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc1e0.
00000100:00000001:1.0:1787829780.349751:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829780.349754:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000001:1.0:1787829780.349755:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829780.349758:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829780.349761:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829780.349764:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829780.349765:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829780.349767:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829780.349769:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349770:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829780.349772:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829780.349773:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829780.349774:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829780.349781:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829780.349784:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829780.349785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829780.349789:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829780.349792:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829780.349794:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1787829780.349802:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1787829780.349803:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829780/real 1787829780]  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829780.349807:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829780.349813:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829780.349817:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000001:0.0:1787829780.349826:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00000001:1.0:1787829780.349826:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829780.349833:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1787829780.349833:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829780.349839:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000010:1.0:1787829780.349840:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829780.349846:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000200:0.0:1787829780.349849:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
02000000:00000001:1.0:1787829780.349849:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829780.349852:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829780.349856:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829780.349858:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829780.349860:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829780.349861:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829780.349866:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:0.0:1787829780.349869:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829780.349871:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829780.349879:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829780.349888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829780.349891:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.349916:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829780.349917:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.349919:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.349922:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829780.349933:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00100000:0.0:1787829780.349934:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829780.349939:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.349952:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829780.349954:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829780.349962:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829780.349982:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829780.349991:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3938)
00000100:00000001:0.0:1787829780.350003:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829780.350007:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350008:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350010:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350013:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829780.350016:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829780.350020:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829780.350028:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829780.350030:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829780.350035:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829780.350044:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327378304:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829780.350049:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350051:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350054:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327378304/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350060:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829780.350062:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350065:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829780.350069:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829780.350071:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829780.350073:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829780.350075:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829780.350078:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829780.350081:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829780.350082:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829780.350084:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829780.350087:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829780.350088:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350091:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350098:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829780.350100:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350102:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829780.350104:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829780.350105:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829780.350109:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350116:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829780.350117:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.350119:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829780.350121:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829780.350123:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829780.350126:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378432:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829780.350129:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829780.350131:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829780.350132:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.350134:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350137:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829780.350139:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829780.350142:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829780.350145:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829780.350148:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829780.350152:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378432, portal 10
00000100:00000001:0.0:1787829780.350154:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829780.350155:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829780.350161:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350167:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829780.350169:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829780.350172:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829780.350174:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327378432, offset 0
00000400:00000200:0.0:1787829780.350178:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829780.350182:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829780.350186:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829780.350189:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829780.350192:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829780.350194:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829780.350197:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829780.350200:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40728:40728:256:4294967295] 192.168.203.116@tcp LPNI seq info [40728:40728:8:4294967295]
00000400:00000200:0.0:1787829780.350204:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829780.350210:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829780.350214:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.350217:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829780.350220:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829780.350223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829780.350228:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829780.350230:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829780.350245:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.350247:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829780.350250:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829780.350254:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829780.350257:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829780.350258:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829780.350261:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829780.350264:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350272:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350275:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350281:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350283:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829780.350286:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829780.350290:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829780.350293:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829780.350295:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298600459
00000100:00000001:0.0:1787829780.350298:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298601133
00000100:00000001:0.0:1787829780.350300:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298600459
00000100:00000001:0.0:1787829780.350302:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298601133
00000100:00000001:0.0:1787829780.350304:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829780.350306:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350308:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829780.350311:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829780/real 1787829780]  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829780.350318:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829780.350319:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829780.350321:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829780.350323:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829780.350325:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829780.350326:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829780.350329:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829780.350338:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350344:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829780.350345:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350346:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350348:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829780.350351:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350357:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829780.350361:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350368:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829780.350370:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350372:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350373:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350374:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829780.350377:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829780.350379:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829780.350382:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829780.350384:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829780.350386:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829780.350391:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378432:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829780.350393:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829780.350395:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350399:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829780.350400:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.350401:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829780.350402:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829780.350404:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829780.350405:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378560:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829780.350407:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829780.350408:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829780.350409:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.350410:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350413:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829780.350414:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829780.350416:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829780.350418:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829780.350420:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829780.350422:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378560, portal 4
00000100:00000001:0.0:1787829780.350423:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829780.350424:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829780.350429:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350435:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829780.350438:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829780.350440:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829780.350442:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327378560, offset 0
00000400:00000200:0.0:1787829780.350446:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829780.350449:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829780.350452:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829780.350455:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829780.350457:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829780.350459:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829780.350461:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829780.350464:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40729:40729:256:4294967295] 192.168.203.116@tcp LPNI seq info [40729:40729:8:4294967295]
00000400:00000200:0.0:1787829780.350468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829780.350474:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829780.350479:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.350482:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829780.350485:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829780.350488:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829780.350492:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829780.350494:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829780.350498:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.350500:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829780.350502:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829780.350506:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829780.350508:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829780.350510:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829780.350512:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829780.350516:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350523:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350527:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350533:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350534:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829780.350537:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829780.350540:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829780.350543:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829780.350545:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298600466
00000100:00000001:0.0:1787829780.350547:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298601133
00000100:00000001:0.0:1787829780.350549:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298600466
00000100:00000001:0.0:1787829780.350551:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298601133
00000100:00000001:0.0:1787829780.350553:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829780.350554:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350556:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829780.350560:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829780/real 1787829780]  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829780.350565:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829780.350566:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829780.350567:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829780.350569:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829780.350570:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829780.350571:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829780.350573:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829780.350578:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350581:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829780.350582:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350583:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350583:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829780.350586:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350590:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829780.350592:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350596:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829780.350597:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350598:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350599:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350600:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829780.350601:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829780.350602:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829780.350604:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829780.350605:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829780.350607:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829780.350611:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378560:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829780.350613:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829780.350615:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350620:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829780.350620:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.350622:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829780.350623:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829780.350624:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829780.350626:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378688:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829780.350627:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829780.350629:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829780.350630:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829780.350631:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350634:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829780.350635:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829780.350637:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829780.350638:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829780.350640:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829780.350641:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378688, portal 4
00000100:00000001:0.0:1787829780.350643:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829780.350644:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:0.0:1787829780.350647:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350651:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829780.350652:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829780.350653:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829780.350671:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327378688, offset 0
00000400:00000200:0.0:1787829780.350674:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829780.350677:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829780.350679:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829780.350681:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829780.350683:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829780.350684:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829780.350686:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829780.350687:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40730:40730:256:4294967295] 192.168.203.116@tcp LPNI seq info [40730:40730:8:4294967295]
00000400:00000200:0.0:1787829780.350690:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829780.350694:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829780.350696:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.350698:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829780.350700:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829780.350702:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829780.350704:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829780.350706:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829780.350708:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829780.350709:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829780.350711:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829780.350713:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829780.350715:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829780.350716:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829780.350717:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829780.350719:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350724:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350726:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350729:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350730:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829780.350732:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829780.350734:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829780.350735:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829780.350736:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298600472
00000100:00000001:0.0:1787829780.350738:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298601133
00000100:00000001:0.0:1787829780.350739:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298600472
00000100:00000001:0.0:1787829780.350740:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298601133
00000100:00000001:0.0:1787829780.350741:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829780.350742:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350743:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829780.350745:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829780/real 1787829780]  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829780.350749:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829780.350750:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829780.350752:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829780.350753:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829780.350754:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829780.350755:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829780.350757:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829780.350762:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350767:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829780.350768:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350769:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350771:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829780.350774:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350780:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829780.350784:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350790:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829780.350792:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350794:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829780.350795:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350796:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829780.350798:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829780.350800:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829780.350803:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829780.350805:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829780.350809:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829780.350817:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378688:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829780.350821:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350823:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350826:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327378432/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350832:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829780.350834:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350836:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829780.350838:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829780.350840:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829780.350841:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829780.350844:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829780.350845:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:0.0:1787829780.350848:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829780.350849:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829780.350850:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829780.350853:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829780.350854:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350856:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350859:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327378560/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350865:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829780.350867:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350868:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829780.350870:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829780.350871:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829780.350872:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829780.350874:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829780.350875:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829780.350878:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829780.350879:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829780.350880:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829780.350882:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829780.350883:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350885:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829780.350888:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327378688/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829780.350893:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829780.350895:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829780.350897:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829780.350899:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829780.350900:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829780.350901:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829780.350903:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829780.350904:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829780.350905:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829780.350906:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829780.350908:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829780.350910:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829780.350911:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350913:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.350916:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829780.350917:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.350919:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829780.350921:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829780.860723:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829780.860730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829780.860733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829780.860738:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829780.860742:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829780.860745:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.860748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829780.860750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829780.860753:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829781.052771:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298601133, at 4298601152
00000100:00000040:3.0:1787829781.052771:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298601133, at 4298601152
00000100:00000010:2.0:1787829781.052786:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:3.0:1787829781.052788:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00010000:00000001:0.0:1787829781.180741:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829781.180746:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829781.180750:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829781.180751:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829781.180753:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829781.180755:0:16416:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829781.180756:0:16416:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829781.180759:0:16416:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829781.180761:0:16416:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829781.180763:0:16416:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829781.180766:0:16416:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829781.180768:0:16416:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829781.180811:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829781.180815:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829781.180818:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829781.884737:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829781.884742:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829781.884743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829781.884747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829781.884749:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829781.884752:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829781.884753:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829781.884755:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829781.884757:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829782.204747:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829782.204753:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829782.204759:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829782.204761:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829782.204764:0:16416:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829782.204766:0:16416:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829782.204782:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829782.204784:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829782.204787:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829782.908812:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829782.908818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829782.908821:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829782.908825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829782.908830:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829782.908833:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829782.908836:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829782.908838:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829782.908841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829783.228738:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829783.228743:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829783.228749:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829783.228751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829783.228754:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829783.228756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829783.228810:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829783.228813:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829783.228817:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829783.932734:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829783.932740:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829783.932742:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829783.932747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829783.932750:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829783.932753:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829783.932756:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829783.932757:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829783.932760:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829784.252753:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829784.252755:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829784.252757:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252761:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829784.252763:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252764:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252765:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252769:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829784.252770:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829784.252771:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829784.252772:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252773:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829784.252775:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252777:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252778:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829784.252780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829784.252781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829784.252808:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829784.252811:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829784.252813:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829784.956762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829784.956767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829784.956768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829784.956772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829784.956775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829784.956778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829784.956780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829784.956781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829784.956783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829785.276731:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829785.276735:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829785.276739:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829785.276739:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829785.276741:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829785.276742:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829785.276754:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829785.276756:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829785.276758:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829785.469001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829785.469008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829785.469010:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829785.469013:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829785.469016:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829785.469023:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3933
00000400:00000200:2.0:1787829785.469031:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829785.469037:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829785.469040:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829785.469044:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469047:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829785.469049:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469051:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829785.469055:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829785.469057:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829785.469059:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829785.469062:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829785.469064:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829785.469067:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829785.469069:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469071:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829785.469073:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469075:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829785.469078:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829785.469082:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469084:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829785.469087:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829785.469108:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829785.469112:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469114:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829785.469117:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829785.469126:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787829785.469130:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829785.469135:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829785.469137:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469140:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829785.469142:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829785.469146:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829785.469154:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829785.469160:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829785.469170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829785.469177:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829785.469180:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829785.469184:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829785.469201:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829785.469204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829785.469206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829785.469208:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829785.469209:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829785.469211:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829785.469212:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3933
00000400:00000200:2.0:1787829785.469215:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829785.469216:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829785.469218:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829785.469220:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469222:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829785.469224:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469226:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829785.469229:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829785.469231:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829785.469233:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829785.469235:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829785.469237:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787829785.469238:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829785.469241:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787829785.469243:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829785.469244:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829785.469249:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829785.469253:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829785.469254:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829785.469259:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469264:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829785.469271:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787829785.469271:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829785.469275:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829785.469276:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469281:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.469282:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829785.469283:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829785.469287:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829785.469289:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829785.469292:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829785.469292:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469296:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:0.0:1787829785.469300:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327378816:192.168.203.116@tcp:250:kworker.0
00000008:00100000:2.0:1787829785.469302:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829785.469307:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1787829785.469308:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.469312:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787829785.469312:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829785.469317:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000020:00000040:2.0:1787829785.469319:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829785.469322:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829785.469324:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829785.469325:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.469327:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787829785.469329:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829785.469332:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829785.469333:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829785.469335:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829785.469338:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829785.469339:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000400:00000200:0.0:1787829785.469344:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829785.469345:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829785.469347:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829785.469353:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00100000:2.0:1787829785.469355:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829785.469359:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378816, portal 25
00000100:00000001:0.0:1787829785.469364:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829785.469364:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829785.469367:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402436160 : -116696307115456 : ffff95dd86115a40)
00000100:00000001:2.0:1787829785.469368:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829785.469379:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829785.469379:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.469406:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829785.469412:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829785.469417:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:2.0:1787829785.469421:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829785.469422:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327378816, offset 0
00000100:00080000:2.0:1787829785.469426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829785.469430:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829785.469433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829785.469436:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829785.469437:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829785.469440:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829785.469440:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829785.469444:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3933
00000400:00000200:0.0:1787829785.469450:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829785.469450:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829785.469456:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829785.469458:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787829785.469460:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829785.469464:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000040:2.0:1787829785.469466:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469469:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829785.469471:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1787829785.469475:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829785.469478:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469480:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829785.469484:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40731:40731:256:4294967295] 192.168.203.116@tcp LPNI seq info [40731:40731:8:4294967295]
00000020:00000001:2.0:1787829785.469486:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829785.469490:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787829785.469490:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829785.469494:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829785.469499:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829785.469502:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1787829785.469504:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829785.469509:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829785.469510:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:0.0:1787829785.469513:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000040:2.0:1787829785.469516:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469519:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1787829785.469522:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829785.469523:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829785.469527:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787829785.469527:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829785.469531:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787829785.469535:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787829785.469538:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.1:1787829785.469539:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787829785.469542:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829785.469546:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829785.469547:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:0.0:1787829785.469550:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000001:2.0:1787829785.469552:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829785.469555:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787829785.469555:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469557:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829785.469561:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000008:00100000:2.0:1787829785.469561:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829785.469565:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000008:00000001:2.0:1787829785.469565:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469567:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829785.469568:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000010:2.0:1787829785.469571:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000100:00000001:0.0:1787829785.469572:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1787829785.469575:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000200:0.0:1787829785.469578:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829785.469579:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829785.469581:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469585:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829785.469587:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829785.469590:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.469592:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:2.0:1787829785.469594:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000040:0.0:1787829785.469596:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829785.469599:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.469605:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.469608:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:2.0:1787829785.469609:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829785.469612:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829785.469618:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829785.469621:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298606252
00000100:00000010:0.0:1787829785.469627:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccb40.
00000100:00000001:2.0:1787829785.469627:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829785.469631:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.469634:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829785.469639:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298606252
00000100:00000040:2.0:1787829785.469639:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1787829785.469643:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc540.
00000100:00000001:0.0:1787829785.469647:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787829785.469654:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829785.469677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829785.469680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829785.469681:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829785.469682:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829785.469685:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829785.469687:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3933
00000400:00000200:2.0:1787829785.469690:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829785.469693:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787829785.469694:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829785.469698:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829785.469699:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829785.469703:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829785.469704:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829785.469705:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829785.469710:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00100000:0.0:1787829785.469711:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829785]  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829785.469712:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829785.469717:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829785.469719:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829785.469722:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000001:0.0:1787829785.469725:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:2.0:1787829785.469725:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829785.469727:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000001:0.0:1787829785.469728:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829785.469731:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000020:00000001:2.0:1787829785.469731:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829785.469737:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000100:00000040:2.0:1787829785.469737:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829785.469739:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:2.0:1787829785.469740:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829785.469743:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1787829785.469745:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469748:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829785.469749:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829785.469751:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829785.469754:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829785.469755:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829785.469756:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829785.469758:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829785.469762:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469766:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829785.469768:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000200:0.0:1787829785.469769:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787829785.469771:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.469775:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829785.469779:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000100:00000001:0.0:1787829785.469780:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829785.469783:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1787829785.469783:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829785.469785:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.469787:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787829785.469788:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829785.469791:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829785.469793:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829785.469794:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829785.469796:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829785.469798:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829785.469803:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:0.0:1787829785.469804:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829785.469809:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829785.469810:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829785.469816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829785.469818:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829785.469823:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.469825:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.469827:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.469830:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829785.469831:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829785.469832:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829785.469835:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829785.469836:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829785.469841:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000040:2.0:1787829785.469841:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.469845:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829785.469851:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829785.469857:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829785.469864:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327378816:192.168.203.116@tcp:250:kworker.0
00000100:00000040:2.0:1787829785.469864:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3943)
00000100:00000001:0.0:1787829785.469871:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.469872:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.469874:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115a40 x1874671327378816/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.469878:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829785.469880:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829785.469881:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829785.469883:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829785.469884:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829785.469885:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829785.469886:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829785.469888:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829785.469890:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829785.469892:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829785.469893:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115a40.
00000100:00000001:0.0:1787829785.469895:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829785.469895:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.469897:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.469901:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829785.469902:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.469903:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829785.469904:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829785.469905:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829785.469907:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.469911:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829785.469912:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.469914:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829785.469915:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829785.469916:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829785.469918:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378944:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829785.469919:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829785.469921:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829785.469922:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.469923:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829785.469925:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829785.469926:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829785.469929:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829785.469930:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829785.469945:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829785.469947:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327378944, portal 10
00000100:00000001:0.0:1787829785.469948:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829785.469949:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:0.0:1787829785.469954:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.469958:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829785.469959:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829785.469961:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829785.469962:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327378944, offset 0
00000400:00000200:0.0:1787829785.469964:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829785.469967:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829785.469970:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829785.469972:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829785.469973:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829785.469975:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829785.469976:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829785.469978:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40732:40732:256:4294967295] 192.168.203.116@tcp LPNI seq info [40732:40732:8:4294967295]
00000400:00000200:0.0:1787829785.469981:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829785.469985:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829785.469987:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829785.469989:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829785.469992:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829785.469993:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829785.469996:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829785.469998:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829785.470000:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829785.470001:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829785.470003:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829785.470005:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829785.470007:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829785.470008:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829785.470009:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829785.470012:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470017:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.470018:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470022:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470023:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829785.470025:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829785.470027:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829785.470029:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829785.470030:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298605579
00000100:00000001:0.0:1787829785.470032:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298606252
00000100:00000001:0.0:1787829785.470033:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298605579
00000100:00000001:0.0:1787829785.470035:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298606252
00000100:00000001:0.0:1787829785.470036:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829785.470037:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470038:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829785.470040:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829785]  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829785.470044:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829785.470045:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829785.470046:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829785.470048:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829785.470048:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829785.470050:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829785.470052:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829785.470057:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470060:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829785.470061:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470062:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.470062:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829785.470064:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470068:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829785.470070:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470074:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829785.470076:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470077:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.470078:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470079:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829785.470080:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829785.470081:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829785.470083:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829785.470084:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829785.470086:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829785.470090:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327378944:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829785.470092:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829785.470094:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470097:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829785.470098:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.470099:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829785.470100:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829785.470101:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829785.470102:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379072:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829785.470104:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829785.470105:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829785.470106:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.470107:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829785.470109:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829785.470110:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829785.470112:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829785.470114:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829785.470116:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829785.470117:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379072, portal 4
00000100:00000001:0.0:1787829785.470118:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829785.470119:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:0.0:1787829785.470122:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470126:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829785.470127:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829785.470128:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829785.470129:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327379072, offset 0
00000400:00000200:0.0:1787829785.470131:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829785.470133:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829785.470135:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829785.470137:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829785.470138:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829785.470140:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829785.470141:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829785.470143:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40733:40733:256:4294967295] 192.168.203.116@tcp LPNI seq info [40733:40733:8:4294967295]
00000400:00000200:0.0:1787829785.470145:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829785.470149:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829785.470151:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829785.470153:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829785.470155:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829785.470156:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829785.470159:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829785.470160:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829785.470162:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829785.470163:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829785.470165:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829785.470167:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829785.470168:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829785.470169:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829785.470171:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829785.470173:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470177:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.470178:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470182:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470183:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829785.470185:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829785.470186:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829785.470188:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829785.470189:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298605586
00000100:00000001:0.0:1787829785.470190:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298606252
00000100:00000001:0.0:1787829785.470192:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298605586
00000100:00000001:0.0:1787829785.470193:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298606252
00000100:00000001:0.0:1787829785.470194:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829785.470195:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470196:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829785.470198:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829785]  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829785.470202:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829785.470202:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829785.470203:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829785.470204:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829785.470205:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829785.470206:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829785.470207:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829785.470212:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470219:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829785.470220:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470221:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.470222:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829785.470223:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470227:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829785.470229:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470232:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829785.470233:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470234:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.470235:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470236:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829785.470237:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829785.470238:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829785.470239:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829785.470241:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829785.470242:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829785.470246:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379072:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829785.470248:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829785.470250:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470254:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829785.470255:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.470256:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829785.470257:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829785.470258:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829785.470260:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379200:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829785.470262:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829785.470263:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829785.470263:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829785.470264:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829785.470266:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829785.470267:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829785.470268:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829785.470270:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829785.470271:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829785.470273:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379200, portal 4
00000100:00000001:0.0:1787829785.470274:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829785.470275:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:0.0:1787829785.470278:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470283:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829785.470285:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829785.470287:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829785.470289:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327379200, offset 0
00000400:00000200:0.0:1787829785.470293:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829785.470296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829785.470299:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829785.470301:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829785.470303:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829785.470305:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829785.470308:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829785.470311:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40734:40734:256:4294967295] 192.168.203.116@tcp LPNI seq info [40734:40734:8:4294967295]
00000400:00000200:0.0:1787829785.470314:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829785.470320:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829785.470325:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829785.470328:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829785.470331:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829785.470333:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829785.470335:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829785.470337:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829785.470339:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829785.470340:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829785.470342:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829785.470344:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829785.470346:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829785.470347:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829785.470349:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829785.470351:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470356:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.470357:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470361:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470362:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829785.470364:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829785.470365:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829785.470367:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829785.470368:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298605591
00000100:00000001:0.0:1787829785.470370:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298606252
00000100:00000001:0.0:1787829785.470371:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298605591
00000100:00000001:0.0:1787829785.470372:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298606252
00000100:00000001:0.0:1787829785.470373:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829785.470374:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470376:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829785.470377:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829785]  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829785.470383:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829785.470385:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829785.470386:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829785.470388:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829785.470390:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829785.470392:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829785.470395:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829785.470402:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470408:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829785.470410:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470411:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.470413:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829785.470416:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470423:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829785.470426:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470430:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829785.470432:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470433:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829785.470433:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470434:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829785.470435:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829785.470437:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829785.470438:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829785.470439:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829785.470442:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829785.470447:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379200:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829785.470449:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470450:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.470452:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327378944/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470455:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829785.470456:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829785.470457:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829785.470459:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829785.470460:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829785.470460:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829785.470462:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829785.470463:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829785.470464:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829785.470465:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829785.470466:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:0.0:1787829785.470468:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829785.470468:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470469:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.470471:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327379072/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470475:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829785.470475:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829785.470476:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829785.470477:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829785.470478:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829785.470478:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829785.470479:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829785.470480:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829785.470481:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829785.470482:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829785.470483:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:0.0:1787829785.470484:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829785.470485:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470486:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829785.470488:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327379200/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829785.470491:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829785.470491:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829785.470492:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829785.470493:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829785.470494:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829785.470494:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829785.470495:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829785.470496:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829785.470497:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829785.470497:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829785.470498:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:0.0:1787829785.470499:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829785.470500:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470501:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829785.470503:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829785.470504:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829785.470505:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829785.470506:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829785.980714:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829785.980719:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829785.980720:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829785.980724:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829785.980726:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829785.980729:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829785.980730:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829785.980732:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829785.980734:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829786.172710:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298606252, at 4298606272
00000100:00000040:3.0:1787829786.172715:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298606252, at 4298606272
00000100:00000010:2.0:1787829786.172726:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00000100:00000010:3.0:1787829786.172727:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00010000:00000001:0.0:1787829786.300742:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829786.300747:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829786.300757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829786.300758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829786.300760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829786.300761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829786.300822:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829786.300825:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829786.300829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829787.004793:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829787.004800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829787.004802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829787.004807:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829787.004811:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829787.004815:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829787.004818:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829787.004821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829787.004824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829787.324774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829787.324781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829787.324790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829787.324792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829787.324796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829787.324797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829787.324841:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829787.324846:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829787.324851:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829788.028724:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829788.028730:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829788.028732:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829788.028735:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829788.028738:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829788.028741:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829788.028743:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829788.028745:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829788.028747:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829788.348758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829788.348764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829788.348768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829788.348769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829788.348772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829788.348773:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829788.348789:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829788.348791:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829788.348793:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829789.052763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829789.052770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829789.052773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829789.052778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829789.052781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829789.052784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829789.052786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829789.052788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829789.052791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829789.372784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829789.372790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829789.372794:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829789.372795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829789.372798:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829789.372799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829789.372869:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829789.372873:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829789.372876:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829790.076719:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829790.076726:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829790.076729:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829790.076735:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829790.076739:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829790.076743:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829790.076746:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829790.076749:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829790.076752:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829790.396812:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829790.396818:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829790.396822:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829790.396824:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829790.396828:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829790.396831:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829790.396833:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829790.396836:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829790.396838:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829790.396840:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829790.396842:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829790.396844:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829790.396910:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829790.396916:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829790.396921:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829790.588807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829790.588820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829790.588823:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829790.588828:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829790.588833:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829790.588842:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3938
00000400:00000200:1.0:1787829790.588852:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829790.588860:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829790.588864:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829790.588867:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.588871:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.588873:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.588875:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829790.588879:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829790.588881:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829790.588884:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829790.588888:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829790.588890:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829790.588892:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829790.588915:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.588917:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.588922:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.588925:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829790.588930:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829790.588935:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829790.588939:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829790.588954:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829790.588959:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829790.588963:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.588967:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829790.588970:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829790.588985:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829790.588990:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829790.588996:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829790.588998:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.589003:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829790.589007:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829790.589013:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829790.589025:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829790.589035:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829790.589049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829790.589060:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829790.589065:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829790.589070:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829790.589095:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829790.589098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829790.589102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829790.589103:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829790.589105:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829790.589107:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829790.589110:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3938
00000400:00000200:1.0:1787829790.589115:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829790.589119:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829790.589121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829790.589125:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589127:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.589130:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589132:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829790.589134:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829790.589135:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829790.589138:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787829790.589138:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829790.589142:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829790.589145:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829790.589146:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787829790.589147:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829790.589153:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829790.589154:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829790.589156:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.589161:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589163:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829790.589167:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:0.0:1787829790.589171:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829790.589172:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589178:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829790.589183:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829790.589192:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829790.589194:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829790.589198:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.589200:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.589203:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829790.589209:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:1.0:1787829790.589224:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829790.589226:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1787829790.589232:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.589235:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.589238:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829790.589244:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327379328:192.168.203.116@tcp:250:kworker.0
00000100:00000010:1.0:1787829790.589248:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829790.589251:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:1.0:1787829790.589255:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829790.589257:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829790.589261:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829790.589263:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829790.589267:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.589269:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829790.589275:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829790.589278:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1787829790.589279:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829790.589282:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.589284:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829790.589293:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000400:00000200:0.0:1787829790.589294:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829790.589301:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829790.589311:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829790.589313:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829790.589325:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829790.589326:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379328, portal 25
00000100:00000001:0.0:1787829790.589334:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829790.589340:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000001:1.0:1787829790.589340:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829790.589345:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829790.589351:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829790.589357:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829790.589365:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829790.589372:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829790.589380:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1787829790.589380:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829790.589385:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1787829790.589389:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829790.589390:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829790.589394:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1787829790.589395:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00080000:1.0:1787829790.589399:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3938
00000100:00000200:0.0:1787829790.589403:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327379328, offset 0
00000400:00000200:1.0:1787829790.589409:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829790.589414:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829790.589419:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829790.589420:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829790.589427:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589429:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.589431:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829790.589436:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829790.589436:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829790.589441:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829790.589444:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829790.589446:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829790.589449:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000040:1.0:1787829790.589450:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829790.589474:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829790.589479:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829790.589486:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:1.0:1787829790.589495:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829790.589497:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1787829790.589509:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829790.589510:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589519:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829790.589526:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40735:40735:256:4294967295] 192.168.203.116@tcp LPNI seq info [40735:40735:8:4294967295]
00000100:00000040:1.0:1787829790.589531:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589540:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829790.589543:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:1.0:1787829790.589546:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829790.589554:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829790.589560:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829790.589585:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829790.589590:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1787829790.589593:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829790.589601:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.589607:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829790.589608:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000008:00100000:1.0:1787829790.589611:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829790.589618:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829790.589619:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829790.589621:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829790.589632:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000800:00000200:0.0:1787829790.589654:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:1.0:1787829790.589656:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.0:1787829790.589661:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
02000000:00000001:1.0:1787829790.589663:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829790.589669:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829790.589673:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829790.589675:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829790.589680:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.1:1787829790.589681:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
02000000:00000001:1.0:1787829790.589685:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829790.589688:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.589691:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000010:1.0:1787829790.589764:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000100:00000001:1.0:1787829790.589788:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829790.589801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829790.589806:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787829790.589825:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829790.589828:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787829790.589851:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829790.589861:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00000040:1.0:1787829790.589867:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829790.589871:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829790.589887:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787829790.589921:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829790.589927:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829790.589956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829790.589963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829790.589967:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829790.589971:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829790.589977:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829790.589983:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3938
00000100:00000001:0.0:1787829790.589984:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:1.0:1787829790.589989:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829790.589995:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829790.589998:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829790.590003:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829790.590010:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.590015:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.590019:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.590023:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829790.590025:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829790.590026:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829790.590028:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829790.590033:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:1.0:1787829790.590033:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829790.590036:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829790.590040:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829790.590042:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829790.590045:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829790.590047:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.590047:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829790.590051:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829790.590055:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829790.590057:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:0.0:1787829790.590062:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829790.590063:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829790.590069:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590071:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298611373
02000000:00000001:1.0:1787829790.590071:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829790.590076:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829790.590079:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.590081:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000010:0.0:1787829790.590083:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc8a0.
00000008:00100000:1.0:1787829790.590085:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829790.590090:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.590092:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590097:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000010:1.0:1787829790.590097:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829790.590104:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829790.590107:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829790.590108:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298611373
02000000:00000001:1.0:1787829790.590109:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829790.590113:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829790.590116:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000010:0.0:1787829790.590117:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc420.
02000000:00000001:1.0:1787829790.590118:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590126:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000010:1.0:1787829790.590130:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829790.590135:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590139:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787829790.590140:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829790.590147:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590156:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829790.590157:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829790.590162:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829790.590172:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829790.590173:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790]  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829790.590204:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829790.590209:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:1.0:1787829790.590223:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3948)
00000400:00000001:0.0:1787829790.590226:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829790.590232:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829790.590236:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829790.590238:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829790.590241:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829790.590246:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829790.590255:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590261:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829790.590263:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590265:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.590267:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829790.590271:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590278:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829790.590282:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590288:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829790.590290:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590292:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.590293:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590295:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829790.590297:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829790.590300:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829790.590305:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829790.590308:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829790.590314:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829790.590324:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327379328:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829790.590331:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590335:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.590339:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327379328/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590349:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829790.590353:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829790.590356:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829790.590360:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829790.590363:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829790.590365:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829790.590368:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829790.590372:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:0.0:1787829790.590376:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829790.590378:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829790.590380:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829790.590383:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829790.590384:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590386:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590394:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829790.590396:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590398:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829790.590400:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829790.590401:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829790.590405:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590411:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829790.590413:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.590415:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829790.590417:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829790.590419:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829790.590421:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379456:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829790.590424:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829790.590426:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829790.590428:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.590430:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829790.590433:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829790.590435:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829790.590439:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829790.590441:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829790.590444:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829790.590447:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379456, portal 10
00000100:00000001:0.0:1787829790.590449:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829790.590450:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829790.590457:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590463:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829790.590465:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829790.590467:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829790.590469:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327379456, offset 0
00000400:00000200:0.0:1787829790.590473:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829790.590476:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829790.590479:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829790.590482:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829790.590485:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829790.590487:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829790.590489:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829790.590492:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40736:40736:256:4294967295] 192.168.203.116@tcp LPNI seq info [40736:40736:8:4294967295]
00000400:00000200:0.0:1787829790.590495:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829790.590501:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829790.590506:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.590508:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829790.590512:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829790.590515:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829790.590519:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829790.590521:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829790.590524:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.590526:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829790.590529:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829790.590532:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829790.590535:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829790.590536:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829790.590538:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829790.590541:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590548:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.590551:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590556:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590557:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829790.590560:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829790.590563:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829790.590566:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829790.590568:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298610699
00000100:00000001:0.0:1787829790.590570:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298611373
00000100:00000001:0.0:1787829790.590572:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298610699
00000100:00000001:0.0:1787829790.590574:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298611373
00000100:00000001:0.0:1787829790.590576:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829790.590577:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590579:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829790.590582:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790]  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829790.590588:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829790.590589:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829790.590590:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829790.590592:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829790.590593:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829790.590595:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829790.590598:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829790.590605:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590611:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829790.590612:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590613:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.590614:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829790.590617:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590622:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829790.590625:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590630:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829790.590632:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590633:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.590634:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590635:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829790.590637:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829790.590639:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829790.590642:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829790.590644:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829790.590647:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829790.590653:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379456:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829790.590678:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829790.590681:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590692:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829790.590693:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.590696:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829790.590697:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829790.590699:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829790.590701:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379584:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829790.590704:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829790.590705:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829790.590707:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.590709:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829790.590712:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829790.590713:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829790.590716:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829790.590719:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829790.590722:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829790.590724:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379584, portal 4
00000100:00000001:0.0:1787829790.590726:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829790.590727:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829790.590732:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590737:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829790.590739:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829790.590741:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829790.590743:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327379584, offset 0
00000400:00000200:0.0:1787829790.590746:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829790.590749:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829790.590752:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829790.590754:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829790.590757:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829790.590758:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829790.590761:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829790.590763:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40737:40737:256:4294967295] 192.168.203.116@tcp LPNI seq info [40737:40737:8:4294967295]
00000400:00000200:0.0:1787829790.590767:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829790.590773:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829790.590777:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.590780:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829790.590783:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829790.590786:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829790.590789:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829790.590791:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829790.590794:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.590796:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829790.590799:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829790.590802:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829790.590804:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829790.590806:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829790.590808:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829790.590811:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590818:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.590820:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590826:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590827:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829790.590830:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829790.590832:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829790.590835:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829790.590837:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298610707
00000100:00000001:0.0:1787829790.590839:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298611373
00000100:00000001:0.0:1787829790.590841:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298610707
00000100:00000001:0.0:1787829790.590843:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298611373
00000100:00000001:0.0:1787829790.590845:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829790.590847:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.590849:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829790.590852:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790]  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829790.590859:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829790.590859:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829790.590861:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829790.590862:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829790.590864:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829790.590865:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829790.590868:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829790.590875:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590880:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829790.590881:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590882:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.590883:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829790.590886:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590891:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829790.590893:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590918:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829790.590920:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590921:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.590922:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.590924:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829790.590925:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829790.590927:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829790.590931:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829790.590932:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829790.590936:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829790.590952:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379584:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829790.590956:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829790.590959:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.590964:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829790.590965:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.590967:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829790.590969:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829790.590971:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829790.590973:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379712:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829790.590975:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829790.590977:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829790.590978:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829790.590979:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829790.590982:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829790.590984:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829790.590986:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829790.590988:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829790.590991:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829790.590994:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379712, portal 4
00000100:00000001:0.0:1787829790.590995:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829790.590996:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:0.0:1787829790.591002:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591007:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829790.591010:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829790.591011:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829790.591014:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327379712, offset 0
00000400:00000200:0.0:1787829790.591017:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829790.591020:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829790.591023:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829790.591026:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829790.591028:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829790.591031:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829790.591032:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829790.591035:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40738:40738:256:4294967295] 192.168.203.116@tcp LPNI seq info [40738:40738:8:4294967295]
00000400:00000200:0.0:1787829790.591038:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829790.591044:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829790.591047:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.591049:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829790.591052:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829790.591055:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829790.591058:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829790.591060:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829790.591064:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829790.591065:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829790.591068:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829790.591071:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829790.591073:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829790.591075:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829790.591077:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829790.591080:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591086:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.591089:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591095:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.591096:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829790.591099:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829790.591102:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829790.591104:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829790.591106:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298610712
00000100:00000001:0.0:1787829790.591108:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298611373
00000100:00000001:0.0:1787829790.591110:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298610712
00000100:00000001:0.0:1787829790.591112:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298611373
00000100:00000001:0.0:1787829790.591113:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829790.591115:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.591116:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829790.591119:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790]  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829790.591125:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829790.591126:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829790.591127:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829790.591129:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829790.591131:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829790.591132:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829790.591135:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829790.591141:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591146:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829790.591148:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591151:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.591152:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829790.591158:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591167:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829790.591173:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591182:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829790.591186:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591188:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829790.591190:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591193:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829790.591195:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829790.591199:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829790.591203:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829790.591206:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829790.591211:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829790.591233:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379712:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829790.591238:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591241:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.591245:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327379456/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591254:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829790.591256:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829790.591259:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829790.591262:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829790.591264:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829790.591266:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829790.591268:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829790.591270:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:0.0:1787829790.591274:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829790.591276:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829790.591278:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829790.591281:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829790.591282:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591284:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.591287:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327379584/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591293:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829790.591294:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829790.591295:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829790.591298:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829790.591299:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829790.591300:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829790.591303:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829790.591304:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4800.
02000000:00000001:0.0:1787829790.591307:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829790.591308:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829790.591310:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829790.591313:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829790.591315:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591317:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829790.591321:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327379712/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829790.591330:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829790.591331:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829790.591334:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829790.591336:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829790.591338:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829790.591340:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829790.591343:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829790.591345:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829790.591348:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829790.591350:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829790.591351:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829790.591354:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829790.591356:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591359:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829790.591363:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829790.591366:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829790.591368:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829790.591370:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829791.100792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829791.100800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829791.100803:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829791.100809:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829791.100813:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829791.100818:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829791.100822:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829791.100825:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829791.100829:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829791.292764:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298611373, at 4298611392
00000100:00000040:3.0:1787829791.292764:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298611373, at 4298611392
00000100:00000010:2.0:1787829791.292794:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000010:3.0:1787829791.292798:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00010000:00000001:0.0:1787829791.420810:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829791.420818:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829791.420823:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829791.420825:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829791.420828:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829791.420829:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829791.420850:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829791.420854:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829791.420857:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829792.124765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829792.124771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829792.124772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829792.124776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829792.124779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829792.124781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829792.124783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829792.124785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829792.124787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829792.444813:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829792.444821:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829792.444826:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829792.444827:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829792.444830:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829792.444831:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829792.444922:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829792.444927:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829792.444931:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829793.148760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829793.148766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829793.148767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829793.148771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829793.148774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829793.148776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829793.148778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829793.148780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829793.148782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829793.468784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829793.468789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829793.468793:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829793.468795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829793.468797:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829793.468799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829793.468826:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829793.468831:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829793.468834:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829794.172766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829794.172772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829794.172774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829794.172778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829794.172781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829794.172784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829794.172786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829794.172789:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829794.172791:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829794.492774:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829794.492776:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829794.492778:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492782:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829794.492784:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492786:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492788:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829794.492791:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829794.492792:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829794.492794:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492795:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829794.492796:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492797:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492798:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829794.492801:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829794.492802:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829794.492816:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829794.492818:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829794.492820:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829795.196777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829795.196782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829795.196784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829795.196788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829795.196791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829795.196794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829795.196797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829795.196799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829795.196801:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829795.516755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829795.516761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829795.516765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829795.516767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829795.516770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829795.516771:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829795.516821:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829795.516824:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829795.516827:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829795.708769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829795.708777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829795.708779:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829795.708782:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829795.708786:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829795.708792:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3943
00000400:00000200:0.0:1787829795.708799:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829795.708805:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829795.708807:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829795.708809:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708812:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.708814:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708815:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829795.708818:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829795.708820:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829795.708822:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829795.708824:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829795.708826:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829795.708827:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829795.708829:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708831:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.708832:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708834:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829795.708836:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829795.708838:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708840:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829795.708842:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829795.708844:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829795.708846:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.708848:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829795.708850:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829795.708857:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716d2c0.
00000020:00000040:0.0:1787829795.708871:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829795.708875:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829795.708876:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.708878:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829795.708880:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829795.708884:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829795.708891:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530000.
00000100:00000001:0.0:1787829795.708896:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829795.708904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829795.708912:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829795.708915:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829795.708918:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c4c777e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829795.708938:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829795.708940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829795.708942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829795.708944:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829795.708945:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829795.708947:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829795.708949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3943
00000400:00000200:0.0:1787829795.708952:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829795.708954:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829795.708956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829795.708959:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708960:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.708963:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708964:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829795.708966:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829795.708967:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829795.708968:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829795.708970:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829795.708971:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:0.0:1787829795.708973:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829795.708976:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829795.708976:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829795.708977:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.708982:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708984:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829795.708984:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1787829795.708989:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:1.0:1787829795.708990:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1787829795.708992:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829795.708996:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829795.708997:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787829795.709002:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829795.709003:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829795.709007:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709012:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829795.709016:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829795.709020:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709023:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829795.709024:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709029:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829795.709030:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f480.
00000020:00000040:0.0:1787829795.709034:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829795.709036:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829795.709037:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829795.709040:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709041:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829795.709041:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829795.709044:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829795.709046:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.709048:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829795.709050:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532000.
00000100:00000001:0.0:1787829795.709054:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829795.709056:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327379840:192.168.203.116@tcp:250:kworker.0
00000100:00100000:0.0:1787829795.709062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829795.709065:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787829795.709071:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787829795.709072:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787829795.709074:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.709077:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829795.709080:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84d4029] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829795.709083:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829795.709089:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829795.709091:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
00000100:00080000:0.0:1787829795.709092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787829795.709098:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829795.709099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829795.709101:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829795.709104:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829795.709106:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829795.709110:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3943
00000400:00000200:1.0:1787829795.709110:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829795.709117:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829795.709119:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000200:0.0:1787829795.709120:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829795.709125:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829795.709127:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.709129:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.709131:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829795.709131:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000001:0.0:1787829795.709137:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829795.709139:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829795.709139:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:0.0:1787829795.709141:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829795.709143:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000200:1.0:1787829795.709143:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379840, portal 25
00000020:00000001:0.0:1787829795.709146:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:1.0:1787829795.709149:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1787829795.709150:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:1.0:1787829795.709154:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:0.0:1787829795.709155:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.709159:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.709162:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829795.709164:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:1.0:1787829795.709168:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829795.709170:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:0.0:1787829795.709177:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829795.709183:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829795.709187:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829795.709191:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829795.709191:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787829795.709195:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709199:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:1.0:1787829795.709200:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000008:00100000:0.0:1787829795.709201:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829795.709205:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829795.709206:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000001:0.0:1787829795.709207:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829795.709212:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716ed00.
00000100:00000200:1.0:1787829795.709214:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327379840, offset 0
00000020:00000040:0.0:1787829795.709216:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:0.0:1787829795.709221:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829795.709222:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709223:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829795.709224:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829795.709227:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.709228:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000010:0.0:1787829795.709231:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531c00.
00000100:00000001:0.0:1787829795.709235:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.709240:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:0.0:1787829795.709241:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829795.709249:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829795.709252:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.709253:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:0.0:1787829795.709261:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008f0359f0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829795.709262:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787829795.709272:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829795.709277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829795.709283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787829795.709286:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:0.0:1787829795.709287:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829795.709291:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829795.709295:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787829795.709297:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829795.709301:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:0.0:1787829795.709304:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3943
00000400:00000200:1.0:1787829795.709312:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40739:40739:256:4294967295] 192.168.203.116@tcp LPNI seq info [40739:40739:8:4294967295]
00000400:00000200:1.0:1787829795.709318:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829795.709325:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829795.709333:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:1.0:1787829795.709337:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787829795.709338:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829795.709348:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829795.709350:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829795.709353:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.709359:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:1.0:1787829795.709359:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00000001:0.0:1787829795.709365:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829795.709368:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829795.709372:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000800:00000200:1.0:1787829795.709373:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:0.0:1787829795.709375:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829795.709379:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:1.0:1787829795.709381:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1787829795.709383:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829795.709388:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:0.0:1787829795.709393:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829795.709394:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829795.709395:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829795.709400:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829795.709402:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787829795.709403:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829795.709411:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000800:00000100:1.0:1787829795.709415:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00080000:0.0:1787829795.709420:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000010:1.0:1787829795.709421:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000100:00000001:0.0:1787829795.709426:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829795.709431:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829795.709431:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787829795.709435:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829795.709440:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709442:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829795.709446:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787829795.709449:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000008:00000001:0.0:1787829795.709450:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709454:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829795.709458:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716de00.
00000400:00000200:1.0:1787829795.709460:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000020:00000040:0.0:1787829795.709461:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000010:1.0:1787829795.709466:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
02000000:00000001:0.0:1787829795.709467:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829795.709469:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829795.709472:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829795.709473:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829795.709475:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709475:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:0.0:1787829795.709481:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533400.
00000100:00000200:1.0:1787829795.709485:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829795.709486:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829795.709496:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829795.709505:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787829795.709511:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787829795.709513:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829795.709515:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829795.709526:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c17e5d1d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709529:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709534:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829795.709541:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.709547:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829795.709548:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709552:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298616492
00000100:00000040:0.0:1787829795.709554:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3953)
00000100:00000010:1.0:1787829795.709561:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63a20.
00000100:00000001:1.0:1787829795.709571:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829795.709577:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298616492
00000100:00000010:1.0:1787829795.709581:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63f00.
00000100:00000001:1.0:1787829795.709584:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829795.709595:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829795.709598:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709601:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829795.709607:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795]  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829795.709618:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829795.709620:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829795.709622:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829795.709625:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829795.709627:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829795.709630:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829795.709634:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829795.709644:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709652:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829795.709677:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.709680:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.709682:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829795.709687:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709695:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829795.709700:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709708:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829795.709711:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.709713:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.709714:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.709717:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829795.709719:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829795.709722:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829795.709726:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829795.709728:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829795.709733:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829795.709743:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327379840:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829795.709748:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.709751:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.709754:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327379840/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709768:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829795.709771:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829795.709774:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829795.709778:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829795.709780:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829795.709781:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829795.709784:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829795.709786:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829795.709790:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829795.709792:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829795.709793:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829795.709797:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829795.709798:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.709800:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.709808:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829795.709810:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.709812:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829795.709814:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829795.709815:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829795.709820:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709827:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829795.709829:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.709831:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829795.709833:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829795.709835:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829795.709838:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379968:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829795.709842:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829795.709844:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829795.709846:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.709848:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829795.709851:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829795.709853:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.709857:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829795.709874:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829795.709879:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829795.709883:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327379968, portal 10
00000100:00000001:1.0:1787829795.709885:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829795.709886:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829795.709894:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.709901:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829795.709904:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829795.709906:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829795.709908:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327379968, offset 0
00000400:00000200:1.0:1787829795.709913:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829795.709918:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829795.709922:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829795.709925:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829795.709928:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829795.709930:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829795.709933:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829795.709937:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40740:40740:256:4294967295] 192.168.203.116@tcp LPNI seq info [40740:40740:8:4294967295]
00000400:00000200:1.0:1787829795.709942:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829795.709948:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829795.709954:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829795.709957:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829795.709961:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829795.709965:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829795.709970:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829795.709973:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829795.709977:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829795.709979:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829795.709982:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829795.709986:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829795.709989:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829795.709991:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829795.709995:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829795.709999:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710007:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.710010:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710017:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710019:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829795.710023:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829795.710026:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829795.710030:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829795.710032:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298615819
00000100:00000001:1.0:1787829795.710035:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298616492
00000100:00000001:1.0:1787829795.710038:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298615819
00000100:00000001:1.0:1787829795.710040:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298616492
00000100:00000001:1.0:1787829795.710042:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829795.710045:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710047:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829795.710051:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795]  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829795.710060:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829795.710061:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829795.710063:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829795.710066:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829795.710068:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829795.710070:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829795.710074:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829795.710083:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710095:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829795.710096:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710098:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.710099:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829795.710103:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710109:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829795.710113:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710120:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829795.710122:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710124:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.710125:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710127:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829795.710129:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829795.710131:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829795.710135:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829795.710137:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829795.710142:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829795.710150:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327379968:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829795.710154:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829795.710158:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710165:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829795.710167:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.710170:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829795.710171:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829795.710174:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829795.710177:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380096:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829795.710180:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829795.710182:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829795.710183:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.710185:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829795.710189:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829795.710191:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.710194:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829795.710197:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829795.710200:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829795.710203:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380096, portal 4
00000100:00000001:1.0:1787829795.710205:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829795.710206:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829795.710211:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710217:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829795.710219:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829795.710221:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829795.710223:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327380096, offset 0
00000400:00000200:1.0:1787829795.710227:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829795.710231:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829795.710234:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829795.710237:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829795.710240:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829795.710242:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829795.710244:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829795.710247:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40741:40741:256:4294967295] 192.168.203.116@tcp LPNI seq info [40741:40741:8:4294967295]
00000400:00000200:1.0:1787829795.710251:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829795.710258:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829795.710263:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829795.710266:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829795.710270:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829795.710273:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829795.710278:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829795.710281:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829795.710285:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829795.710286:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829795.710289:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829795.710293:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829795.710296:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829795.710297:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829795.710300:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829795.710303:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710310:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.710313:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710320:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710322:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829795.710324:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829795.710327:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829795.710331:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829795.710333:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298615826
00000100:00000001:1.0:1787829795.710336:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298616492
00000100:00000001:1.0:1787829795.710338:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298615826
00000100:00000001:1.0:1787829795.710340:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298616492
00000100:00000001:1.0:1787829795.710343:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829795.710345:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710347:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829795.710350:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795]  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829795.710359:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829795.710360:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829795.710362:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829795.710365:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829795.710367:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829795.710369:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829795.710372:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829795.710382:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710388:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829795.710390:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710392:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.710393:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829795.710397:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710404:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829795.710409:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710414:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829795.710415:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710417:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.710417:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710419:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829795.710420:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829795.710421:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829795.710423:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829795.710424:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829795.710427:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829795.710433:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380096:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829795.710436:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829795.710438:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710442:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829795.710442:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.710444:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829795.710445:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829795.710447:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829795.710449:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380224:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829795.710451:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829795.710452:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829795.710453:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829795.710454:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829795.710457:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829795.710458:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829795.710460:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829795.710462:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829795.710464:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829795.710466:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380224, portal 4
00000100:00000001:1.0:1787829795.710468:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829795.710469:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829795.710473:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710477:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829795.710478:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829795.710480:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829795.710482:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327380224, offset 0
00000400:00000200:1.0:1787829795.710485:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829795.710488:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829795.710490:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829795.710492:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829795.710494:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829795.710496:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829795.710497:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829795.710499:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40742:40742:256:4294967295] 192.168.203.116@tcp LPNI seq info [40742:40742:8:4294967295]
00000400:00000200:1.0:1787829795.710502:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829795.710506:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829795.710509:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829795.710511:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829795.710513:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829795.710516:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829795.710519:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829795.710520:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829795.710523:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829795.710524:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829795.710526:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829795.710528:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829795.710531:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829795.710532:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829795.710533:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829795.710535:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710540:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.710542:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710547:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710548:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829795.710550:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829795.710552:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829795.710554:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829795.710556:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298615831
00000100:00000001:1.0:1787829795.710558:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298616492
00000100:00000001:1.0:1787829795.710559:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298615831
00000100:00000001:1.0:1787829795.710561:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298616492
00000100:00000001:1.0:1787829795.710562:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829795.710563:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710564:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829795.710566:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795]  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829795.710571:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829795.710571:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829795.710573:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829795.710574:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829795.710575:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829795.710577:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829795.710579:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829795.710584:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710587:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829795.710588:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710589:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.710590:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829795.710592:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710596:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829795.710598:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710603:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829795.710605:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710606:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829795.710607:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710608:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829795.710609:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829795.710611:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829795.710613:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829795.710614:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829795.710616:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829795.710621:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380224:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829795.710623:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710624:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.710626:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327379968/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710629:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829795.710631:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829795.710632:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829795.710633:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829795.710634:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829795.710635:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829795.710636:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829795.710637:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829795.710639:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829795.710640:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829795.710641:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829795.710643:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829795.710644:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710645:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.710646:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327380096/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710670:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829795.710671:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829795.710672:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829795.710674:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829795.710675:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829795.710676:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829795.710677:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829795.710678:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531c00.
02000000:00000001:1.0:1787829795.710679:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829795.710680:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829795.710681:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829795.710682:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829795.710683:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710684:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829795.710686:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327380224/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829795.710691:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829795.710691:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829795.710692:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829795.710694:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829795.710694:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829795.710695:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829795.710696:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829795.710697:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:1.0:1787829795.710698:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829795.710699:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829795.710700:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829795.710701:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829795.710702:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710703:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829795.710705:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829795.710706:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829795.710707:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829795.710708:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829796.220772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829796.220778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829796.220779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829796.220783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829796.220786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829796.220789:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829796.220791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829796.220793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829796.220795:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829796.412783:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298616492, at 4298616512
00000100:00000040:3.0:1787829796.412783:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298616492, at 4298616512
00000100:00000010:3.0:1787829796.412806:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f00.
00000100:00000010:2.0:1787829796.412813:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a20.
00010000:00000001:0.0:1787829796.540781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829796.540791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829796.540798:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829796.540799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829796.540801:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829796.540803:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829796.540843:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829796.540848:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829796.540853:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829797.244806:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829797.244814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829797.244816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829797.244821:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829797.244825:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829797.244828:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829797.244831:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829797.244834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829797.244837:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829797.564774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829797.564780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829797.564784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829797.564785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829797.564787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829797.564789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829797.564805:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829797.564808:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829797.564811:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829798.268809:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829798.268815:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829798.268817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829798.268821:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829798.268825:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829798.268828:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829798.268831:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829798.268834:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829798.268837:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829798.588787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829798.588794:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829798.588799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829798.588801:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829798.588804:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829798.588805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829798.588909:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829798.588913:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829798.588918:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829799.292751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829799.292757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829799.292758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829799.292763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829799.292765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829799.292768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829799.292770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829799.292772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829799.292774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829799.612746:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829799.612751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829799.612756:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829799.612757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829799.612760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829799.612761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829799.612796:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829799.612799:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829799.612802:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829800.316787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829800.316794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829800.316796:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829800.316802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829800.316805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829800.316809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829800.316811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829800.316814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829800.316817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829800.636746:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829800.636751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829800.636755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829800.636757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829800.636760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829800.636763:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829800.636765:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829800.636767:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829800.636770:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829800.636772:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829800.636773:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829800.636775:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829800.636793:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829800.636795:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829800.636797:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829800.828736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829800.828745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829800.828747:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829800.828750:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829800.828755:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829800.828761:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3948
00000400:00000200:0.0:1787829800.828768:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829800.828773:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829800.828776:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829800.828778:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828781:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829800.828783:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828784:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829800.828787:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829800.828789:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829800.828791:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829800.828793:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829800.828795:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829800.828796:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829800.828798:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828799:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829800.828801:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828802:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829800.828804:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829800.828807:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828809:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829800.828811:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829800.828813:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829800.828815:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.828817:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829800.828819:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829800.828844:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716e580.
00000020:00000040:0.0:1787829800.828847:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829800.828850:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829800.828851:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.828853:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829800.828855:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829800.828859:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829800.828865:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531400.
00000100:00000001:0.0:1787829800.828871:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829800.828879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829800.828887:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829800.828890:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829800.828894:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc9f2c41] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829800.828914:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829800.828917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829800.828919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829800.828920:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829800.828922:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829800.828923:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829800.828925:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3948
00000400:00000200:0.0:1787829800.828927:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829800.828929:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829800.828931:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829800.828933:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828935:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829800.828936:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828937:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829800.828939:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829800.828940:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829800.828941:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829800.828943:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829800.828944:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:0.0:1787829800.828945:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829800.828948:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828950:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829800.828951:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828953:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829800.828955:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:0.0:1787829800.828958:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829800.828960:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829800.828961:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829800.828963:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1787829800.828966:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829800.828969:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.828970:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829800.828971:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829800.828974:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:0.0:1787829800.828976:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829800.828980:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.828983:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829800.828985:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f0c0.
00000100:00000040:1.0:1787829800.828985:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829800.828989:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829800.828993:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829800.828995:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.828999:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829800.829001:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829800.829004:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829009:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829800.829012:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.829018:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:0.0:1787829800.829020:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829800.829022:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829800.829028:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.829029:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829800.829035:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327380352:192.168.203.116@tcp:250:kworker.0
00000100:00100000:0.0:1787829800.829041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829800.829042:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829800.829047:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829800.829050:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.829054:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829800.829058:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829800.829062:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829800.829062:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829800.829066:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829800.829072:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003ce953a3] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829800.829076:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829800.829084:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829800.829087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1787829800.829092:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000100:00080000:0.0:1787829800.829093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829800.829097:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829800.829100:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1787829800.829101:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000001:0.0:1787829800.829108:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1787829800.829111:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380352, portal 25
00000100:00080000:0.0:1787829800.829112:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3948
00000100:00000001:1.0:1787829800.829116:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829800.829119:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000400:00000200:0.0:1787829800.829123:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829800.829129:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829800.829131:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829800.829137:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829800.829144:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829800.829145:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.829151:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829800.829152:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829800.829156:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000040:0.0:1787829800.829159:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:1.0:1787829800.829161:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327380352, offset 0
00000100:00000001:0.0:1787829800.829165:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829800.829170:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787829800.829171:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:0.0:1787829800.829174:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:0.0:1787829800.829177:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829800.829179:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:0.0:1787829800.829181:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829800.829183:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:0.0:1787829800.829186:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829800.829191:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:0.0:1787829800.829194:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.829199:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829800.829199:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1787829800.829206:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829800.829208:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:0.0:1787829800.829211:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829800.829214:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:0.0:1787829800.829216:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:1.0:1787829800.829219:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:0.0:1787829800.829224:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829800.829228:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40743:40743:256:4294967295] 192.168.203.116@tcp LPNI seq info [40743:40743:8:4294967295]
00000100:00000001:0.0:1787829800.829230:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829800.829235:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829800.829235:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:0.0:1787829800.829240:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829800.829247:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829800.829250:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787829800.829252:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829800.829257:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.0:1787829800.829259:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000008:00000001:0.0:1787829800.829264:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787829800.829266:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00000001:0.0:1787829800.829268:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829800.829275:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716c000.
00000800:00000200:1.0:1787829800.829275:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829800.829281:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000020:00000040:0.0:1787829800.829282:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:0.0:1787829800.829290:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.1:1787829800.829292:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:0.0:1787829800.829293:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.829299:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.1:1787829800.829299:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
02000000:00000001:0.0:1787829800.829301:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829800.829305:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787829800.829306:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.829310:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
02000000:00000010:0.0:1787829800.829312:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533000.
00000400:00000200:1.0:1787829800.829315:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787829800.829318:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829800.829322:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829800.829327:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000100:00100000:0.0:1787829800.829329:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787829800.829330:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829800.829336:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829800.829343:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829800.829345:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829800.829350:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829358:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829800.829362:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cdd0ad0f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829800.829365:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829395:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829800.829396:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829398:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1787829800.829401:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829800.829404:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829800.829410:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787829800.829411:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829800.829415:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787829800.829416:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298621612
00000100:00080000:0.0:1787829800.829419:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829800.829422:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:1.0:1787829800.829423:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d80.
00000100:00080000:0.0:1787829800.829427:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3948
00000100:00000001:1.0:1787829800.829432:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000400:00000200:0.0:1787829800.829433:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829800.829441:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829800.829441:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298621612
00000100:00080000:0.0:1787829800.829447:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1787829800.829448:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63b40.
00000100:00000040:0.0:1787829800.829456:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829800.829456:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829800.829460:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829800.829462:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829800.829464:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829800.829467:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829800.829469:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1787829800.829473:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:1.0:1787829800.829477:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800]  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787829800.829478:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829800.829481:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:0.0:1787829800.829485:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829800.829489:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000400:00000001:1.0:1787829800.829492:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:0.0:1787829800.829493:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000001:1.0:1787829800.829494:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829800.829498:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000020:00000001:0.0:1787829800.829499:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:1.0:1787829800.829502:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829800.829505:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000040:0.0:1787829800.829507:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829800.829508:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787829800.829513:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1787829800.829515:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829800.829519:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829800.829525:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829800.829531:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000200:1.0:1787829800.829536:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829800.829539:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829800.829547:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829800.829551:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829800.829551:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829800.829554:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829800.829557:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829800.829557:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.829560:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829800.829563:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829800.829566:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829571:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829800.829575:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829800.829577:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829800.829581:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829800.829583:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:0.0:1787829800.829587:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829588:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829592:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829800.829593:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829594:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829600:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000010:0.0:1787829800.829602:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829800.829602:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:0.0:1787829800.829610:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00080000:1.0:1787829800.829610:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829800.829616:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
02000000:00000001:0.0:1787829800.829617:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829800.829621:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829622:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1787829800.829623:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829800.829625:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829800.829628:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829800.829632:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829800.829633:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532400.
00000100:00000001:0.0:1787829800.829638:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829800.829646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829800.829672:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829800.829674:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829800.829677:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000226d20b5] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829800.829684:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829800.829687:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3958)
00000100:00100000:1.0:1787829800.829688:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327380352:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829800.829696:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829698:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.829703:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327380352/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829708:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829800.829711:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829800.829713:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829800.829715:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829800.829716:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829800.829718:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829800.829720:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829800.829722:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829800.829724:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829800.829726:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829800.829727:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829800.829729:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829800.829730:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829732:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829737:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829800.829738:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829739:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829800.829740:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829800.829742:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829800.829744:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829753:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829800.829754:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.829756:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829800.829757:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829800.829759:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829800.829761:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380480:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829800.829763:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829800.829764:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829800.829765:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.829766:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829800.829769:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829800.829770:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829800.829773:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829800.829775:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829800.829777:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829800.829780:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380480, portal 10
00000100:00000001:1.0:1787829800.829781:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829800.829782:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829800.829787:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829791:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829800.829793:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829800.829794:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829800.829796:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327380480, offset 0
00000400:00000200:1.0:1787829800.829799:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829800.829802:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829800.829804:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829800.829806:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829800.829808:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829800.829810:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829800.829812:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829800.829814:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40744:40744:256:4294967295] 192.168.203.116@tcp LPNI seq info [40744:40744:8:4294967295]
00000400:00000200:1.0:1787829800.829817:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829800.829822:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829800.829838:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.829842:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829800.829845:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829800.829847:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829800.829849:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829800.829863:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829800.829867:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.829868:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829800.829870:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829800.829873:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829800.829876:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829800.829877:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829800.829880:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829800.829885:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829893:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.829896:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829903:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829905:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829800.829909:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829800.829913:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829800.829917:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829800.829919:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298620939
00000100:00000001:1.0:1787829800.829922:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298621612
00000100:00000001:1.0:1787829800.829924:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298620939
00000100:00000001:1.0:1787829800.829927:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298621612
00000100:00000001:1.0:1787829800.829929:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829800.829932:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.829933:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829800.829937:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800]  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829800.829943:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829800.829944:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829800.829945:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829800.829947:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829800.829948:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829800.829950:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829800.829952:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829800.829957:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829961:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829800.829962:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829963:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.829964:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829800.829967:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829971:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829800.829973:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.829978:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829800.829979:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829980:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.829981:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.829982:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829800.829984:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829800.829985:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829800.829988:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829800.829989:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829800.829992:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829800.829997:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380480:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829800.829999:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829800.830001:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830005:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829800.830006:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.830007:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829800.830008:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829800.830010:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829800.830011:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380608:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829800.830013:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829800.830014:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829800.830016:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.830017:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829800.830019:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829800.830020:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829800.830023:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829800.830024:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829800.830026:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829800.830028:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380608, portal 4
00000100:00000001:1.0:1787829800.830030:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829800.830030:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829800.830034:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830039:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829800.830041:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829800.830042:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829800.830044:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327380608, offset 0
00000400:00000200:1.0:1787829800.830046:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829800.830049:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829800.830052:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829800.830054:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829800.830056:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829800.830057:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829800.830059:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829800.830061:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40745:40745:256:4294967295] 192.168.203.116@tcp LPNI seq info [40745:40745:8:4294967295]
00000400:00000200:1.0:1787829800.830064:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829800.830068:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829800.830071:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.830073:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829800.830076:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829800.830078:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829800.830081:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829800.830083:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829800.830085:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.830087:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829800.830089:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829800.830091:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829800.830093:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829800.830094:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829800.830096:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829800.830098:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830103:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.830105:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830110:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.830111:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829800.830113:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829800.830115:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829800.830117:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829800.830119:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298620946
00000100:00000001:1.0:1787829800.830120:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298621612
00000100:00000001:1.0:1787829800.830122:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298620946
00000100:00000001:1.0:1787829800.830123:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298621612
00000100:00000001:1.0:1787829800.830124:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829800.830125:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.830126:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829800.830129:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800]  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829800.830133:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829800.830134:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829800.830135:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829800.830137:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829800.830138:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829800.830139:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829800.830141:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829800.830146:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830150:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829800.830151:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830152:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.830152:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829800.830155:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830158:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829800.830161:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830166:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829800.830167:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830168:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.830169:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830170:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829800.830171:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829800.830173:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829800.830174:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829800.830176:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829800.830178:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829800.830182:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380608:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829800.830185:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829800.830187:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830191:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829800.830191:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.830193:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829800.830194:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829800.830195:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829800.830197:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380736:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829800.830198:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829800.830199:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829800.830200:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829800.830201:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829800.830203:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829800.830205:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829800.830207:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829800.830209:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829800.830210:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829800.830212:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380736, portal 4
00000100:00000001:1.0:1787829800.830213:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829800.830214:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829800.830218:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830221:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829800.830223:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829800.830224:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829800.830226:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327380736, offset 0
00000400:00000200:1.0:1787829800.830228:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829800.830230:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829800.830232:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829800.830234:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829800.830236:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829800.830237:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829800.830239:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829800.830241:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40746:40746:256:4294967295] 192.168.203.116@tcp LPNI seq info [40746:40746:8:4294967295]
00000400:00000200:1.0:1787829800.830243:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829800.830247:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829800.830250:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.830252:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829800.830254:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829800.830256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787829800.830258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829800.830260:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787829800.830263:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829800.830264:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829800.830265:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829800.830267:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829800.830269:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829800.830270:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829800.830272:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829800.830274:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830278:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.830280:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830285:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.830286:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829800.830287:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829800.830289:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829800.830291:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829800.830303:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298620951
00000100:00000001:1.0:1787829800.830308:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298621612
00000100:00000001:1.0:1787829800.830309:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298620951
00000100:00000001:1.0:1787829800.830311:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298621612
00000100:00000001:1.0:1787829800.830312:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829800.830314:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.830315:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829800.830318:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800]  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829800.830322:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829800.830323:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829800.830324:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829800.830326:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829800.830327:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829800.830328:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829800.830330:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829800.830336:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830344:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829800.830345:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830346:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.830347:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829800.830350:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830353:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829800.830356:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830360:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829800.830361:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830362:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829800.830363:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830365:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829800.830366:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829800.830369:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829800.830372:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829800.830375:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829800.830379:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829800.830387:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380736:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829800.830391:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830393:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.830397:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327380480/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830404:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829800.830405:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829800.830408:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829800.830411:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829800.830412:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829800.830414:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829800.830416:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829800.830418:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829800.830421:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829800.830422:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829800.830423:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829800.830425:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829800.830425:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830427:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.830429:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327380608/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830434:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829800.830435:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829800.830436:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829800.830437:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829800.830438:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829800.830439:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829800.830440:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829800.830441:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829800.830442:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829800.830443:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829800.830443:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829800.830445:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829800.830445:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830446:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829800.830448:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327380736/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829800.830452:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829800.830453:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829800.830453:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829800.830455:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829800.830455:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829800.830456:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829800.830457:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829800.830458:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829800.830460:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829800.830460:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829800.830461:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829800.830463:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829800.830463:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830464:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829800.830467:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829800.830468:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829800.830469:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829800.830470:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829801.340709:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829801.340714:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829801.340715:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829801.340720:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829801.340723:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829801.340725:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829801.340727:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829801.340729:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829801.340731:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829801.532727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298621612, at 4298621632
00000100:00000040:3.0:1787829801.532727:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298621612, at 4298621632
00000100:00000010:2.0:1787829801.532743:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d80.
00000100:00000010:3.0:1787829801.532745:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63b40.
00010000:00000001:0.0:1787829801.660764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829801.660770:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829801.660776:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829801.660777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829801.660780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829801.660781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829801.660845:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829801.660849:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829801.660853:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829802.364756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829802.364763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829802.364765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829802.364769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829802.364772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829802.364776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829802.364778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829802.364780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829802.364783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829802.684735:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829802.684741:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829802.684746:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829802.684747:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829802.684749:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829802.684750:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829802.684788:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829802.684792:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829802.684796:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829803.388746:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829803.388751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829803.388753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829803.388768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829803.388770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829803.388773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829803.388775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829803.388777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829803.388780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829803.708777:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829803.708786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829803.708791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829803.708792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829803.708794:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829803.708795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829803.708825:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829803.708828:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829803.708831:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829804.412762:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829804.412767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829804.412769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829804.412773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829804.412776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829804.412779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829804.412781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829804.412782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829804.412784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732729:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829804.732736:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829804.732738:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829804.732740:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732744:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829804.732745:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732747:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732748:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732751:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829804.732752:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829804.732753:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829804.732754:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732755:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829804.732756:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732758:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732759:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829804.732761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829804.732762:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829804.732830:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829804.732835:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829804.732839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829805.436743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829805.436748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829805.436750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829805.436754:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829805.436757:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829805.436760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829805.436761:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829805.436764:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829805.436766:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829805.756789:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829805.756809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829805.756815:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829805.756816:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829805.756819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829805.756820:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829805.756862:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829805.756866:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829805.756869:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829805.948729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829805.948737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829805.948740:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829805.948743:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829805.948748:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829805.948754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3953
00000400:00000200:2.0:1787829805.948761:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829805.948765:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829805.948768:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829805.948770:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948772:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829805.948774:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948776:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829805.948779:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829805.948781:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829805.948783:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829805.948785:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829805.948787:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829805.948800:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829805.948802:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948804:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829805.948805:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948806:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829805.948809:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829805.948811:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948813:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829805.948815:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829805.948818:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829805.948820:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.948821:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829805.948823:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829805.948830:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829805.948833:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829805.948836:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829805.948838:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.948840:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829805.948842:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829805.948845:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829805.948851:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829805.948856:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829805.948864:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829805.948872:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829805.948875:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829805.948878:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829805.948894:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829805.948897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829805.948899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829805.948900:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829805.948901:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829805.948904:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829805.948905:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3953
00000400:00000200:2.0:1787829805.948908:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829805.948910:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829805.948911:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829805.948914:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948916:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829805.948917:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948919:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829805.948921:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829805.948923:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829805.948924:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829805.948926:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829805.948929:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829805.948932:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829805.948936:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829805.948937:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829805.948940:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829805.948943:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787829805.948944:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829805.948948:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787829805.948949:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829805.948954:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:1.0:1787829805.948959:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829805.948960:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829805.948967:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829805.948972:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829805.948978:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829805.948980:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829805.948980:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829805.948987:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.948989:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787829805.948990:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829805.948993:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:2.0:1787829805.948994:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829805.948998:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787829805.948999:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949004:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829805.949008:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327380864:192.168.203.116@tcp:250:kworker.0
00000100:00000010:2.0:1787829805.949009:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829805.949013:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:1.0:1787829805.949017:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829805.949017:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829805.949020:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949022:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829805.949024:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829805.949024:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829805.949026:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.949028:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829805.949030:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
02000000:00000001:1.0:1787829805.949033:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787829805.949035:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829805.949040:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
00000100:00100000:2.0:1787829805.949042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787829805.949045:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949053:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829805.949056:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829805.949057:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829805.949063:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787829805.949064:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829805.949073:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000001:2.0:1787829805.949080:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829805.949082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1787829805.949083:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380864, portal 25
00000100:00080000:2.0:1787829805.949085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829805.949089:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829805.949089:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829805.949090:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829805.949093:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787829805.949094:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829805.949098:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3953
00000400:00000200:2.0:1787829805.949104:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829805.949106:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829805.949108:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829805.949113:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829805.949118:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949122:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829805.949127:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787829805.949127:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949131:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:1.0:1787829805.949135:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829805.949135:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829805.949136:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829805.949140:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787829805.949142:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000020:00000040:2.0:1787829805.949142:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829805.949145:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829805.949148:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:1.0:1787829805.949150:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327380864, offset 0
00000100:00000040:2.0:1787829805.949151:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949156:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829805.949157:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949159:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829805.949164:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787829805.949164:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829805.949170:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949172:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829805.949173:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829805.949177:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829805.949177:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829805.949181:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949186:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787829805.949191:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787829805.949193:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:2.0:1787829805.949194:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829805.949198:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949200:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949201:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000010:2.0:1787829805.949204:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000400:00000200:1.0:1787829805.949208:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:2.0:1787829805.949208:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829805.949213:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1787829805.949215:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:2.0:1787829805.949216:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949220:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829805.949221:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829805.949223:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949225:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40747:40747:256:4294967295] 192.168.203.116@tcp LPNI seq info [40747:40747:8:4294967295]
02000000:00000010:2.0:1787829805.949227:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829805.949232:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:2.0:1787829805.949238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829805.949245:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829805.949249:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949252:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1787829805.949255:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1787829805.949266:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949266:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829805.949272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829805.949275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000010:1.0:1787829805.949276:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00000001:2.0:1787829805.949276:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829805.949279:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829805.949282:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829805.949286:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3953
00000800:00000200:1.0:1787829805.949289:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000400:00000200:2.0:1787829805.949292:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000200:1.0:1787829805.949297:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000400:00000200:2.0:1787829805.949297:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829805.949302:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829805.949306:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829805.949310:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829805.949314:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787829805.949314:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1787829805.949319:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949322:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000100:1.0:1787829805.949325:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829805.949325:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829805.949328:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829805.949330:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:1.0:1787829805.949331:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000020:00000040:2.0:1787829805.949332:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829805.949336:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829805.949337:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:1.0:1787829805.949339:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1787829805.949341:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829805.949344:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829805.949349:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829805.949350:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829805.949351:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829805.949356:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829805.949359:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829805.949360:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000100:00000001:2.0:1787829805.949361:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829805.949365:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:1.0:1787829805.949367:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
02000000:00000001:2.0:1787829805.949367:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829805.949372:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949373:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829805.949374:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:2.0:1787829805.949375:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829805.949379:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949380:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829805.949384:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787829805.949384:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829805.949388:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829805.949393:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829805.949395:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829805.949399:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829805.949401:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829805.949403:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829805.949406:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:1.0:1787829805.949408:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829805.949410:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829805.949415:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829805.949416:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829805.949441:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829805.949444:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.949447:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829805.949449:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949451:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829805.949457:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.949458:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829805.949461:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3963)
00000400:00000200:1.0:1787829805.949466:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829805.949470:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298626732
00000100:00000010:1.0:1787829805.949476:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63480.
00000100:00000001:1.0:1787829805.949483:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829805.949488:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298626732
00000100:00000010:1.0:1787829805.949492:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d20.
00000100:00000001:1.0:1787829805.949495:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829805.949505:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829805.949508:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.949510:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829805.949515:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805]  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829805.949525:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829805.949527:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829805.949530:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829805.949534:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829805.949535:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829805.949537:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829805.949541:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829805.949547:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949552:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829805.949553:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949554:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.949556:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829805.949559:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949564:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829805.949567:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949572:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829805.949573:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949574:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.949575:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949577:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829805.949578:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829805.949580:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829805.949583:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829805.949584:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829805.949587:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829805.949592:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327380864:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829805.949595:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949596:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.949598:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327380864/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949603:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829805.949604:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829805.949606:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829805.949608:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829805.949610:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829805.949611:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829805.949612:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829805.949614:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:1.0:1787829805.949616:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829805.949618:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829805.949619:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:1.0:1787829805.949621:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829805.949622:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949624:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949628:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829805.949630:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.949631:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829805.949632:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829805.949633:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829805.949635:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949640:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829805.949642:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.949644:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829805.949646:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829805.949648:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829805.949651:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380992:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829805.949683:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829805.949685:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829805.949687:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.949689:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829805.949692:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829805.949694:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949698:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829805.949700:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829805.949705:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829805.949707:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327380992, portal 10
00000100:00000001:1.0:1787829805.949709:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829805.949710:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:1.0:1787829805.949717:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949723:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829805.949725:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829805.949728:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829805.949730:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327380992, offset 0
00000400:00000200:1.0:1787829805.949734:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829805.949738:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829805.949742:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829805.949745:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829805.949747:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829805.949750:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829805.949752:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829805.949755:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40748:40748:256:4294967295] 192.168.203.116@tcp LPNI seq info [40748:40748:8:4294967295]
00000400:00000200:1.0:1787829805.949759:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829805.949765:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829805.949769:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829805.949772:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829805.949776:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829805.949778:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829805.949783:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829805.949786:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829805.949804:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829805.949806:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829805.949809:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829805.949812:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829805.949816:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829805.949817:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829805.949820:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829805.949823:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949831:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.949833:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949839:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.949841:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829805.949845:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829805.949849:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829805.949851:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829805.949854:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298626059
00000100:00000001:1.0:1787829805.949856:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298626732
00000100:00000001:1.0:1787829805.949858:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298626059
00000100:00000001:1.0:1787829805.949861:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298626732
00000100:00000001:1.0:1787829805.949863:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829805.949865:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.949867:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829805.949871:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805]  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829805.949879:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829805.949880:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829805.949882:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829805.949885:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829805.949887:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829805.949889:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829805.949892:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829805.949900:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949905:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829805.949907:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949909:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.949910:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829805.949913:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949921:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829805.949925:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949933:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829805.949936:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949937:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.949938:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.949939:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829805.949941:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829805.949942:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829805.949945:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829805.949946:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829805.949949:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829805.949954:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327380992:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829805.949956:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829805.949958:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949962:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829805.949963:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.949965:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829805.949966:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829805.949968:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829805.949970:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381120:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829805.949972:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829805.949973:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829805.949974:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.949976:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829805.949978:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829805.949979:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.949982:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829805.949984:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829805.949986:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829805.949988:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381120, portal 4
00000100:00000001:1.0:1787829805.949989:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829805.949990:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:1.0:1787829805.949994:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.949999:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829805.950000:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829805.950002:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829805.950003:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327381120, offset 0
00000400:00000200:1.0:1787829805.950006:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829805.950008:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829805.950011:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829805.950013:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829805.950015:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829805.950016:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829805.950018:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829805.950020:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40749:40749:256:4294967295] 192.168.203.116@tcp LPNI seq info [40749:40749:8:4294967295]
00000400:00000200:1.0:1787829805.950022:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829805.950027:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829805.950030:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829805.950032:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829805.950035:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829805.950037:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829805.950039:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829805.950041:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829805.950044:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829805.950045:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829805.950047:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829805.950049:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829805.950051:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829805.950053:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829805.950054:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829805.950057:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950062:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.950064:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950068:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.950069:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829805.950071:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829805.950073:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829805.950075:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829805.950076:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298626066
00000100:00000001:1.0:1787829805.950078:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298626732
00000100:00000001:1.0:1787829805.950080:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298626066
00000100:00000001:1.0:1787829805.950081:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298626732
00000100:00000001:1.0:1787829805.950082:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829805.950083:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.950085:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829805.950087:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805]  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829805.950091:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829805.950092:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829805.950093:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829805.950095:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829805.950096:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829805.950097:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829805.950099:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829805.950104:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950112:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829805.950113:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950114:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.950115:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829805.950117:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950121:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829805.950123:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950127:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829805.950129:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950130:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.950131:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950132:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829805.950133:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829805.950134:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829805.950136:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829805.950137:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829805.950139:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829805.950144:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381120:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829805.950146:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829805.950148:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950152:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829805.950153:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.950155:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829805.950156:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829805.950158:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829805.950159:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381248:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829805.950161:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829805.950162:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829805.950163:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829805.950164:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829805.950167:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829805.950168:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829805.950169:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829805.950171:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829805.950173:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829805.950175:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381248, portal 4
00000100:00000001:1.0:1787829805.950176:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829805.950177:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:1.0:1787829805.950180:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950184:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829805.950186:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829805.950187:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829805.950188:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327381248, offset 0
00000400:00000200:1.0:1787829805.950191:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829805.950193:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829805.950194:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829805.950196:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829805.950198:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829805.950199:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829805.950201:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829805.950203:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40750:40750:256:4294967295] 192.168.203.116@tcp LPNI seq info [40750:40750:8:4294967295]
00000400:00000200:1.0:1787829805.950206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829805.950210:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829805.950213:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829805.950215:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829805.950217:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829805.950219:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829805.950221:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829805.950223:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829805.950225:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829805.950226:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829805.950228:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829805.950230:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829805.950232:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829805.950233:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829805.950234:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829805.950237:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950241:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.950243:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950247:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.950248:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829805.950249:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829805.950251:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829805.950253:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829805.950254:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298626071
00000100:00000001:1.0:1787829805.950256:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298626732
00000100:00000001:1.0:1787829805.950257:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298626071
00000100:00000001:1.0:1787829805.950259:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298626732
00000100:00000001:1.0:1787829805.950260:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829805.950261:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.950262:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829805.950264:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805]  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829805.950270:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829805.950271:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829805.950272:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829805.950273:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829805.950274:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829805.950275:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829805.950277:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829805.950282:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950285:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829805.950286:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950287:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.950288:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829805.950290:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950294:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829805.950296:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950300:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829805.950302:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950303:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829805.950303:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950305:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829805.950306:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829805.950308:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829805.950310:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829805.950311:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829805.950314:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829805.950319:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381248:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829805.950321:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950322:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.950324:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327380992/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950328:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829805.950329:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829805.950330:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829805.950331:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829805.950332:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829805.950333:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829805.950334:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829805.950335:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:1.0:1787829805.950337:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829805.950338:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829805.950339:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:1.0:1787829805.950340:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829805.950341:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950342:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.950344:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327381120/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950348:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829805.950349:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829805.950350:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829805.950351:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829805.950352:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829805.950352:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829805.950353:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829805.950354:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:1.0:1787829805.950355:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829805.950357:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829805.950357:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:1.0:1787829805.950359:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829805.950360:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950361:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829805.950363:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327381248/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829805.950367:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829805.950367:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829805.950368:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829805.950369:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829805.950370:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829805.950370:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829805.950372:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829805.950372:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:1.0:1787829805.950374:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829805.950374:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829805.950375:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:1.0:1787829805.950376:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829805.950377:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950378:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829805.950380:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829805.950381:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829805.950383:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829805.950384:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829806.460772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829806.460780:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829806.460782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829806.460806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829806.460810:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829806.460814:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829806.460817:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829806.460820:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829806.460823:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829806.652760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298626732, at 4298626752
00000100:00000040:3.0:1787829806.652760:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298626732, at 4298626752
00000100:00000010:2.0:1787829806.652776:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63480.
00000100:00000010:3.0:1787829806.652779:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d20.
00010000:00000001:0.0:1787829806.780738:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829806.780745:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829806.780751:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829806.780752:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829806.780756:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829806.780757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829806.780774:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829806.780776:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829806.780779:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829807.484718:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829807.484723:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829807.484725:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829807.484729:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829807.484732:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829807.484734:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829807.484736:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829807.484738:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829807.484740:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829807.804758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829807.804763:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829807.804767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829807.804768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829807.804771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829807.804772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829807.804850:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829807.804855:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829807.804859:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829808.508798:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829808.508804:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829808.508806:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829808.508810:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829808.508814:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829808.508817:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829808.508820:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829808.508822:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829808.508825:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829808.828753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829808.828760:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829808.828766:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829808.828768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829808.828785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829808.828787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829808.828833:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829808.828837:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829808.828840:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829809.532731:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829809.532736:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829809.532737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829809.532741:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829809.532743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829809.532746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829809.532748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829809.532750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829809.532752:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829809.852759:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829809.852776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829809.852781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829809.852782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829809.852785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829809.852786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829809.852804:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829809.852806:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829809.852808:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829810.556750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829810.556768:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829810.556770:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829810.556775:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829810.556777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829810.556781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829810.556783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829810.556784:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829810.556787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829810.876801:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829810.876808:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829810.876814:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829810.876815:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829810.876820:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829810.876823:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829810.876826:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829810.876829:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829810.876832:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829810.876835:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829810.876837:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829810.876840:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829810.876887:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829810.876892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829810.876896:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829811.068765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829811.068774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829811.068776:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829811.068780:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829811.068783:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829811.068789:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3958
00000400:00000200:2.0:1787829811.068797:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829811.068802:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829811.068806:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829811.068808:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068810:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829811.068812:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068814:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829811.068817:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829811.068818:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829811.068820:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829811.068823:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829811.068824:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829811.068826:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829811.068827:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068829:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829811.068830:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068831:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829811.068834:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829811.068836:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068838:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829811.068840:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829811.068842:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829811.068844:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.068846:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829811.068847:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829811.068855:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829811.068857:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829811.068860:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829811.068861:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.068863:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829811.068865:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829811.068869:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829811.068874:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829811.068879:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829811.068887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829811.068895:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829811.068898:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829811.068902:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829811.068919:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829811.068921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829811.068924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829811.068925:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829811.068926:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829811.068928:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829811.068929:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3958
00000400:00000200:2.0:1787829811.068932:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829811.068933:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829811.068941:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829811.068944:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068945:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829811.068948:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068950:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829811.068951:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829811.068952:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829811.068953:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787829811.068954:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787829811.068955:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829811.068959:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829811.068960:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787829811.068961:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829811.068964:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829811.068965:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068968:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829811.068971:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829811.068973:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829811.068974:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829811.068979:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829811.068984:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829811.068989:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.068993:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787829811.068993:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829811.068996:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829811.068998:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829811.069000:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829811.069001:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.069003:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829811.069004:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829811.069008:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787829811.069010:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:1.0:1787829811.069014:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327381376:192.168.203.116@tcp:250:kworker.0
00000008:00000001:2.0:1787829811.069015:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069019:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069021:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1787829811.069023:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
02000000:00000001:1.0:1787829811.069025:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:2.0:1787829811.069027:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829811.069029:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829811.069030:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829811.069032:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787829811.069033:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069036:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829811.069037:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:1.0:1787829811.069038:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:2.0:1787829811.069039:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.069042:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829811.069045:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000400:00000200:1.0:1787829811.069050:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829811.069051:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829811.069054:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000100:00100000:2.0:1787829811.069058:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787829811.069062:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000001:2.0:1787829811.069067:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787829811.069070:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381376, portal 25
00000100:00000001:2.0:1787829811.069070:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829811.069074:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069075:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829811.069080:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000100:00000001:2.0:1787829811.069086:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829811.069088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829811.069092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:1.0:1787829811.069094:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829811.069094:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829811.069096:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829811.069101:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829811.069106:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3958
00000100:00000001:1.0:1787829811.069115:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829811.069120:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829811.069121:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829811.069126:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000400:00000200:2.0:1787829811.069130:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000200:1.0:1787829811.069133:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327381376, offset 0
00000100:00080000:2.0:1787829811.069133:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829811.069139:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.069141:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829811.069145:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787829811.069146:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.069149:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829811.069151:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829811.069151:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829811.069153:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829811.069155:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:1.0:1787829811.069156:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787829811.069160:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829811.069163:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829811.069165:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:2.0:1787829811.069169:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829811.069172:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829811.069173:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1787829811.069176:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829811.069180:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829811.069184:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829811.069184:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829811.069187:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000040:2.0:1787829811.069189:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:1.0:1787829811.069194:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40751:40751:256:4294967295] 192.168.203.116@tcp LPNI seq info [40751:40751:8:4294967295]
00000100:00080000:2.0:1787829811.069196:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829811.069200:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829811.069203:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829811.069204:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829811.069206:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829811.069209:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069212:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829811.069216:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787829811.069218:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000008:00000001:2.0:1787829811.069220:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069224:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829811.069228:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000010:2.0:1787829811.069229:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000020:00000040:2.0:1787829811.069234:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829811.069236:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829811.069236:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069237:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829811.069238:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829811.069239:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829811.069242:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829811.069243:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787829811.069245:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00100000:2.0:1787829811.069249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1787829811.069256:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829811.069260:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:1.0:1787829811.069263:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787829811.069263:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829811.069269:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.1:1787829811.069274:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829811.069280:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1787829811.069284:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787829811.069288:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829811.069289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:1.0:1787829811.069293:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000100:00080000:2.0:1787829811.069294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829811.069297:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829811.069298:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829811.069299:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787829811.069300:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829811.069307:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1787829811.069307:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3958
00000400:00000200:2.0:1787829811.069313:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829811.069316:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829811.069318:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787829811.069320:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000100:00000040:2.0:1787829811.069321:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829811.069324:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:2.0:1787829811.069324:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829811.069329:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829811.069331:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787829811.069332:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829811.069335:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829811.069337:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000200:1.0:1787829811.069339:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829811.069339:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829811.069342:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829811.069345:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829811.069350:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829811.069356:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829811.069358:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829811.069359:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829811.069363:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829811.069366:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829811.069366:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829811.069371:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829811.069378:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829811.069380:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069382:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069384:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787829811.069385:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829811.069389:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829811.069391:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829811.069391:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069397:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829811.069399:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787829811.069400:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829811.069404:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298631852
00000100:00000010:1.0:1787829811.069408:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63000.
00000008:00000001:2.0:1787829811.069409:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069412:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829811.069415:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000100:00000001:1.0:1787829811.069416:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000020:00000040:2.0:1787829811.069417:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829811.069421:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829811.069422:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069423:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298631852
00000100:00000001:2.0:1787829811.069424:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829811.069426:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829811.069429:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829811.069432:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e639c0.
02000000:00000010:2.0:1787829811.069433:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829811.069439:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069440:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00100000:2.0:1787829811.069444:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829811.069447:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829811.069454:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.069457:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829811.069459:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829811.069460:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829811.069466:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829811.069473:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829811.069476:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3968)
00000100:00100000:1.0:1787829811.069478:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829811/real 1787829811]  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829811.069489:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829811.069491:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829811.069493:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829811.069497:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829811.069498:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829811.069501:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829811.069505:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829811.069511:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069516:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829811.069517:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069518:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.069519:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829811.069522:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069526:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829811.069530:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069535:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829811.069537:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069538:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.069539:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069541:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829811.069542:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829811.069544:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829811.069546:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829811.069547:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829811.069550:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829811.069555:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327381376:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829811.069560:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069562:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.069564:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327381376/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069568:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829811.069570:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829811.069572:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829811.069573:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829811.069575:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829811.069576:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829811.069577:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829811.069579:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:1.0:1787829811.069583:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829811.069584:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829811.069587:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:1.0:1787829811.069590:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829811.069591:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069594:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069600:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829811.069602:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069604:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829811.069606:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829811.069608:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829811.069612:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069621:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829811.069622:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.069624:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829811.069626:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829811.069629:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829811.069632:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381504:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829811.069634:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829811.069636:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829811.069637:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.069638:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829811.069641:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829811.069642:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829811.069645:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829811.069647:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829811.069649:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829811.069652:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381504, portal 10
00000100:00000001:1.0:1787829811.069653:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829811.069654:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:1.0:1787829811.069689:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069694:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829811.069705:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829811.069719:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829811.069721:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327381504, offset 0
00000400:00000200:1.0:1787829811.069724:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829811.069727:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829811.069730:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829811.069732:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829811.069734:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829811.069743:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829811.069745:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829811.069747:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40752:40752:256:4294967295] 192.168.203.116@tcp LPNI seq info [40752:40752:8:4294967295]
00000400:00000200:1.0:1787829811.069750:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829811.069768:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829811.069771:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829811.069774:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829811.069777:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829811.069779:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829811.069782:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829811.069784:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829811.069786:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829811.069788:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829811.069790:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829811.069803:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829811.069805:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829811.069807:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829811.069808:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829811.069811:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069817:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.069819:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069823:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069824:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829811.069826:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829811.069829:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829811.069831:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829811.069833:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298631179
00000100:00000001:1.0:1787829811.069835:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298631852
00000100:00000001:1.0:1787829811.069836:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298631179
00000100:00000001:1.0:1787829811.069838:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298631852
00000100:00000001:1.0:1787829811.069839:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829811.069840:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.069842:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829811.069844:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829811/real 1787829811]  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829811.069849:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829811.069850:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829811.069852:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829811.069853:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829811.069854:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829811.069856:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829811.069858:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829811.069864:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069868:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829811.069869:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069870:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.069871:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829811.069873:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069877:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829811.069879:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069883:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829811.069885:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069886:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.069886:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.069888:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829811.069889:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829811.069890:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829811.069893:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829811.069894:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829811.069897:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829811.069902:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381504:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829811.069904:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829811.069906:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069915:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829811.069916:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.069917:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829811.069919:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829811.069920:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829811.069922:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381632:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829811.069924:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829811.069925:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829811.069926:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.069927:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829811.069929:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829811.069930:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829811.069932:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829811.069934:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829811.069936:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829811.069938:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381632, portal 4
00000100:00000001:1.0:1787829811.069939:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829811.069940:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:1.0:1787829811.069944:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.069948:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829811.069949:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829811.069951:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829811.069952:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327381632, offset 0
00000400:00000200:1.0:1787829811.069954:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829811.069957:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829811.069959:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829811.069961:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829811.069962:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829811.069964:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829811.069965:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829811.069968:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40753:40753:256:4294967295] 192.168.203.116@tcp LPNI seq info [40753:40753:8:4294967295]
00000400:00000200:1.0:1787829811.069971:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829811.069975:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829811.069978:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829811.069980:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829811.069982:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829811.069984:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829811.069987:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829811.069988:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829811.069991:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829811.069992:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829811.069994:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829811.069997:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829811.069998:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829811.069999:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829811.070001:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829811.070003:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070008:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.070010:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070014:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.070015:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829811.070017:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829811.070019:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829811.070020:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829811.070021:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298631186
00000100:00000001:1.0:1787829811.070023:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298631852
00000100:00000001:1.0:1787829811.070024:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298631186
00000100:00000001:1.0:1787829811.070026:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298631852
00000100:00000001:1.0:1787829811.070028:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829811.070029:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.070030:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829811.070033:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829811/real 1787829811]  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829811.070038:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829811.070039:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829811.070040:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829811.070041:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829811.070042:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829811.070043:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829811.070045:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829811.070051:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070054:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829811.070055:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070056:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.070057:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829811.070059:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070063:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829811.070065:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070069:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829811.070071:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070072:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.070072:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070073:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829811.070074:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829811.070076:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829811.070078:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829811.070080:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829811.070082:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829811.070087:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381632:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829811.070089:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829811.070091:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070095:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829811.070096:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.070097:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829811.070098:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829811.070099:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829811.070101:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829811.070102:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829811.070103:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829811.070104:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829811.070105:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829811.070107:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829811.070108:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829811.070110:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829811.070111:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829811.070113:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829811.070115:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381760, portal 4
00000100:00000001:1.0:1787829811.070116:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829811.070117:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:1.0:1787829811.070120:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070124:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829811.070126:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829811.070127:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829811.070129:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327381760, offset 0
00000400:00000200:1.0:1787829811.070131:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829811.070133:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829811.070135:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829811.070137:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829811.070139:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829811.070140:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829811.070142:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829811.070144:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40754:40754:256:4294967295] 192.168.203.116@tcp LPNI seq info [40754:40754:8:4294967295]
00000400:00000200:1.0:1787829811.070147:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829811.070151:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829811.070153:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829811.070155:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829811.070157:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829811.070159:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787829811.070162:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829811.070163:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787829811.070166:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829811.070167:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829811.070169:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829811.070171:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829811.070172:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829811.070173:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829811.070175:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829811.070177:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070181:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.070183:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070187:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.070189:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829811.070191:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829811.070192:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829811.070194:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829811.070196:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298631191
00000100:00000001:1.0:1787829811.070197:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298631852
00000100:00000001:1.0:1787829811.070199:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298631191
00000100:00000001:1.0:1787829811.070200:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298631852
00000100:00000001:1.0:1787829811.070201:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829811.070202:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.070203:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829811.070206:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829811/real 1787829811]  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829811.070210:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829811.070211:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829811.070211:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829811.070213:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829811.070214:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829811.070215:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829811.070217:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829811.070222:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070226:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829811.070227:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070228:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.070229:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829811.070231:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070235:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829811.070237:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070241:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829811.070243:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070244:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829811.070245:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070246:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829811.070247:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829811.070248:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829811.070250:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829811.070251:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829811.070254:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829811.070258:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327381760:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829811.070260:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070261:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.070263:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327381504/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070267:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829811.070268:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829811.070269:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829811.070271:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829811.070272:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829811.070272:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829811.070274:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829811.070275:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:1.0:1787829811.070276:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829811.070277:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829811.070278:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:1.0:1787829811.070280:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829811.070281:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070282:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.070284:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327381632/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070288:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829811.070289:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829811.070290:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829811.070291:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829811.070292:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829811.070292:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829811.070294:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829811.070294:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:1.0:1787829811.070296:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829811.070296:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829811.070297:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:1.0:1787829811.070298:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829811.070299:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070300:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829811.070302:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327381760/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829811.070305:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829811.070306:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829811.070307:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829811.070308:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829811.070309:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829811.070309:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829811.070310:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829811.070311:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:1.0:1787829811.070312:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829811.070313:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829811.070314:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:1.0:1787829811.070315:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829811.070316:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070317:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829811.070319:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829811.070320:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829811.070321:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829811.070323:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829811.580728:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829811.580733:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829811.580735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829811.580740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829811.580743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829811.580746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829811.580748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829811.580760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829811.580763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829811.772824:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298631852, at 4298631872
00000100:00000040:3.0:1787829811.772825:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298631852, at 4298631872
00000100:00000010:2.0:1787829811.772845:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63000.
00000100:00000010:3.0:1787829811.772850:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e639c0.
00010000:00000001:0.0:1787829811.900797:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829811.900803:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829811.900809:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829811.900810:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829811.900813:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829811.900814:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829811.900860:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829811.900865:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829811.900869:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829812.604808:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829812.604816:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829812.604818:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829812.604823:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829812.604827:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829812.604831:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829812.604834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829812.604837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829812.604840:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829812.924754:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829812.924759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829812.924764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829812.924765:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829812.924768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829812.924769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829812.924803:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829812.924805:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829812.924808:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829813.628712:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829813.628718:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829813.628719:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829813.628723:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829813.628726:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829813.628729:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829813.628731:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829813.628733:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829813.628745:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829813.948764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829813.948769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829813.948774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829813.948775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829813.948778:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829813.948779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829813.948839:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829813.948843:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829813.948845:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829814.652821:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829814.652827:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829814.652828:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829814.652833:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829814.652836:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829814.652839:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829814.652841:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829814.652842:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829814.652845:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829814.972787:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829814.972790:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829814.972792:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972797:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829814.972799:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972801:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972802:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972805:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829814.972806:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829814.972807:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829814.972808:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972810:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829814.972810:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972811:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972813:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829814.972815:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829814.972816:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829814.972861:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829814.972866:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829814.972870:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829815.676790:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829815.676796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829815.676797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829815.676802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829815.676805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829815.676807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829815.676810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829815.676812:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829815.676814:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829815.996775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829815.996781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829815.996785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829815.996787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829815.996790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829815.996791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829815.996826:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829815.996828:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829815.996830:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829816.188760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829816.188769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829816.188771:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829816.188774:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829816.188778:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829816.188785:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3963
00000400:00000200:3.0:1787829816.188797:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829816.188803:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829816.188806:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829816.188808:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188811:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.188812:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188814:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829816.188817:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829816.188818:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829816.188820:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829816.188823:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829816.188825:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829816.188827:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829816.188829:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188831:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.188832:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188834:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829816.188836:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829816.188839:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188840:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829816.188842:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829816.188845:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829816.188847:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.188848:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829816.188850:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829816.188857:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0000.
00000020:00000040:3.0:1787829816.188859:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829816.188862:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829816.188864:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.188866:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829816.188868:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829816.188871:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829816.188876:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829816.188881:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829816.188888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829816.188896:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829816.188900:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829816.188903:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e3ac96] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829816.188920:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829816.188923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829816.188925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829816.188926:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829816.188927:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829816.188929:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829816.188931:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3963
00000400:00000200:3.0:1787829816.188934:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829816.188936:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829816.188937:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829816.188939:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188940:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.188942:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188943:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829816.188944:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829816.188945:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829816.188946:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829816.188947:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829816.188949:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829816.188950:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829816.188952:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188954:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.188955:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188957:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829816.188959:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829816.188961:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829816.188962:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829816.188964:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829816.188968:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829816.188970:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.188971:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829816.188974:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00100000:3.0:1787829816.188975:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829816.188979:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.188981:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787829816.188981:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829816.188986:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed2d00.
00000100:00000001:1.0:1787829816.188987:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787829816.188989:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829816.188993:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829816.188994:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.188997:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829816.188998:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:1.0:1787829816.188999:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829816.189000:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829816.189005:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829816.189009:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829816.189017:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829816.189019:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829816.189022:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189028:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787829816.189030:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787829816.189031:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.189035:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787829816.189036:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009ef4d7da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829816.189042:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189049:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829816.189050:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327381888:192.168.203.116@tcp:250:kworker.0
00000100:00080000:3.0:1787829816.189051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829816.189057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829816.189060:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787829816.189061:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787829816.189062:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829816.189065:0:37:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:1.0:1787829816.189067:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1787829816.189068:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3963
02000000:00000001:1.0:1787829816.189073:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787829816.189078:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829816.189079:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:3.0:1787829816.189085:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
02000000:00000010:1.0:1787829816.189087:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
00000100:00080000:3.0:1787829816.189088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000001:1.0:1787829816.189093:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829816.189095:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189100:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829816.189104:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829816.189105:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189110:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:1.0:1787829816.189111:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000020:00000001:3.0:1787829816.189113:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829816.189117:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829816.189119:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787829816.189121:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000020:00000040:3.0:1787829816.189122:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829816.189127:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829816.189129:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829816.189132:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:1.0:1787829816.189133:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327381888, portal 25
00000100:00000001:3.0:1787829816.189134:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.189136:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829816.189141:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787829816.189141:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829816.189146:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400053760 : -116696309497856 : ffff95dd85ed0000)
00000020:00000040:3.0:1787829816.189146:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829816.189150:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189154:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829816.189158:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787829816.189159:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829816.189162:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829816.189168:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189173:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829816.189176:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829816.189180:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189182:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.189185:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:3.0:1787829816.189189:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed21c0.
00000100:00000040:1.0:1787829816.189190:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829816.189194:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000020:00000040:3.0:1787829816.189195:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000200:1.0:1787829816.189200:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327381888, offset 0
02000000:00000001:3.0:1787829816.189200:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829816.189203:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189206:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829816.189207:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787829816.189210:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829816.189211:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829816.189217:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829816.189219:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.189221:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00100000:3.0:1787829816.189226:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829816.189233:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829816.189236:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829816.189239:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.189243:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1787829816.189244:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007805199e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829816.189252:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829816.189254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829816.189257:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829816.189257:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829816.189259:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829816.189262:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:3.0:1787829816.189263:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829816.189267:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3963
00000400:00000200:1.0:1787829816.189271:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787829816.189272:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.189278:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:3.0:1787829816.189279:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829816.189282:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787829816.189286:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40755:40755:256:4294967295] 192.168.203.116@tcp LPNI seq info [40755:40755:8:4294967295]
00000100:00000040:3.0:1787829816.189288:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189292:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.189297:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829816.189299:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829816.189299:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829816.189304:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829816.189305:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829816.189306:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829816.189308:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829816.189309:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829816.189311:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829816.189316:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189319:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829816.189322:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829816.189325:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787829816.189325:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829816.189329:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829816.189334:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189337:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829816.189338:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829816.189340:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829816.189342:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189343:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829816.189344:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.0:1787829816.189348:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000008:00000001:3.0:1787829816.189349:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189353:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829816.189357:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0780.
00000800:00000010:1.0:1787829816.189358:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000020:00000040:3.0:1787829816.189359:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829816.189363:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829816.189364:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829816.189365:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829816.189367:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:1.0:1787829816.189368:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:3.0:1787829816.189370:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829816.189374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
02000000:00000010:3.0:1787829816.189375:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829816.189381:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829816.189385:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:3.0:1787829816.189390:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.1:1787829816.189391:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000100:00000001:3.0:1787829816.189401:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000100:1.0:1787829816.189403:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829816.189404:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787829816.189408:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000100:00000040:3.0:1787829816.189410:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4586439] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829816.189418:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.189419:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787829816.189422:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3973)
00000400:00000200:1.0:1787829816.189428:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829816.189435:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829816.189438:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829816.189445:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829816.189449:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189456:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.189459:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189464:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.189465:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829816.189470:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.189473:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829816.189476:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298636972
00000100:00000010:1.0:1787829816.189480:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e639c0.
00000100:00000001:1.0:1787829816.189486:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829816.189490:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298636972
00000100:00000010:1.0:1787829816.189493:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63000.
00000100:00000001:1.0:1787829816.189495:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829816.189503:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829816.189505:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.189507:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829816.189510:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829816/real 1787829816]  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829816.189517:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829816.189519:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829816.189522:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829816.189525:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829816.189526:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829816.189528:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829816.189530:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829816.189536:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189540:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829816.189541:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189543:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.189544:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829816.189548:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189556:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829816.189568:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189580:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829816.189584:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189586:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.189588:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189591:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829816.189593:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829816.189596:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829816.189600:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829816.189602:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829816.189605:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829816.189611:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327381888:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829816.189614:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189616:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.189618:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0000 x1874671327381888/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189622:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829816.189624:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829816.189626:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829816.189628:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829816.189629:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829816.189631:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829816.189632:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829816.189634:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:1.0:1787829816.189637:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829816.189638:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829816.189639:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0000.
00000100:00000001:1.0:1787829816.189641:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829816.189642:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189644:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189649:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829816.189650:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.189652:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829816.189653:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829816.189654:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829816.189682:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189691:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829816.189692:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.189694:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829816.189695:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829816.189697:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829816.189699:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382016:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829816.189701:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829816.189703:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829816.189704:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.189705:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829816.189708:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829816.189709:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.189712:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829816.189714:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829816.189717:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829816.189734:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382016, portal 10
00000100:00000001:1.0:1787829816.189736:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829816.189737:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400065280 : -116696309486336 : ffff95dd85ed2d00)
00000100:00000040:1.0:1787829816.189742:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189747:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829816.189749:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829816.189751:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829816.189753:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327382016, offset 0
00000400:00000200:1.0:1787829816.189756:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829816.189759:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.189762:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829816.189764:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829816.189766:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829816.189768:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829816.189769:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829816.189772:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40756:40756:256:4294967295] 192.168.203.116@tcp LPNI seq info [40756:40756:8:4294967295]
00000400:00000200:1.0:1787829816.189775:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.189780:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829816.189783:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829816.189785:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829816.189788:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829816.189790:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829816.189793:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829816.189807:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829816.189810:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829816.189812:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829816.189814:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829816.189817:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829816.189821:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829816.189823:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829816.189826:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829816.189830:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189838:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.189841:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189849:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.189851:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829816.189854:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829816.189858:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829816.189862:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829816.189865:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298636299
00000100:00000001:1.0:1787829816.189868:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298636972
00000100:00000001:1.0:1787829816.189870:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298636299
00000100:00000001:1.0:1787829816.189880:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298636972
00000100:00000001:1.0:1787829816.189882:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829816.189886:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.189888:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829816.189891:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829816/real 1787829816]  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829816.189897:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829816.189898:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829816.189900:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829816.189902:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829816.189903:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829816.189905:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829816.189907:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829816.189913:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189917:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829816.189918:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189919:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.189920:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829816.189922:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189926:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829816.189929:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189933:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829816.189935:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189936:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.189937:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.189938:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829816.189939:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829816.189941:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829816.189944:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829816.189945:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829816.189948:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829816.189953:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382016:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829816.189955:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829816.189957:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189961:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829816.189962:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.189963:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829816.189965:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829816.189966:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829816.189968:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382144:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829816.189970:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829816.189971:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829816.189973:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.189974:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829816.189977:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829816.189978:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.189981:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829816.189982:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829816.189985:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829816.189987:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382144, portal 4
00000100:00000001:1.0:1787829816.189988:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829816.189989:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400062400 : -116696309489216 : ffff95dd85ed21c0)
00000100:00000040:1.0:1787829816.189993:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.189997:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829816.189999:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829816.190001:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829816.190003:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327382144, offset 0
00000400:00000200:1.0:1787829816.190005:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829816.190008:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.190011:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829816.190013:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829816.190015:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829816.190016:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829816.190018:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829816.190020:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40757:40757:256:4294967295] 192.168.203.116@tcp LPNI seq info [40757:40757:8:4294967295]
00000400:00000200:1.0:1787829816.190023:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.190028:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829816.190031:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829816.190033:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829816.190036:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829816.190038:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829816.190041:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829816.190042:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829816.190045:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829816.190046:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829816.190049:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829816.190051:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829816.190053:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829816.190054:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829816.190055:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829816.190057:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190062:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.190065:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190069:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.190070:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829816.190072:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829816.190075:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829816.190077:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829816.190078:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298636306
00000100:00000001:1.0:1787829816.190080:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298636972
00000100:00000001:1.0:1787829816.190081:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298636306
00000100:00000001:1.0:1787829816.190083:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298636972
00000100:00000001:1.0:1787829816.190084:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829816.190085:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.190086:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829816.190088:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829816/real 1787829816]  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829816.190093:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829816.190094:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829816.190095:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829816.190097:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829816.190098:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829816.190099:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829816.190101:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829816.190106:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190110:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829816.190111:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190112:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.190113:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829816.190115:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190119:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829816.190121:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190126:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829816.190128:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190129:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.190130:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190131:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829816.190132:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829816.190134:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829816.190135:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829816.190137:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829816.190139:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829816.190143:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382144:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829816.190146:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829816.190148:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190151:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829816.190152:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.190153:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829816.190154:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829816.190156:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829816.190157:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829816.190159:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829816.190160:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829816.190161:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829816.190162:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829816.190164:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829816.190166:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829816.190168:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829816.190170:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000400:00000010:1.0:1787829816.190171:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000100:00000200:1.0:1787829816.190173:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382272, portal 4
00000100:00000001:1.0:1787829816.190174:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829816.190175:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400055680 : -116696309495936 : ffff95dd85ed0780)
00000100:00000040:1.0:1787829816.190179:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190183:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829816.190184:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829816.190186:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000200:1.0:1787829816.190187:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327382272, offset 0
00000400:00000200:1.0:1787829816.190190:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829816.190192:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.190194:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829816.190195:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829816.190197:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829816.190199:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829816.190200:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829816.190202:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40758:40758:256:4294967295] 192.168.203.116@tcp LPNI seq info [40758:40758:8:4294967295]
00000400:00000200:1.0:1787829816.190205:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829816.190209:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829816.190212:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829816.190213:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000800:00000200:1.0:1787829816.190215:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829816.190217:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787829816.190222:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829816.190224:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787829816.190228:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829816.190231:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:1.0:1787829816.190233:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829816.190237:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829816.190240:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:1.0:1787829816.190241:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:1.0:1787829816.190244:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829816.190248:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190257:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.190260:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190268:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.190270:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829816.190273:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829816.190276:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829816.190279:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829816.190281:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298636311
00000100:00000001:1.0:1787829816.190284:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298636972
00000100:00000001:1.0:1787829816.190285:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298636311
00000100:00000001:1.0:1787829816.190287:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298636972
00000100:00000001:1.0:1787829816.190288:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829816.190289:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.190291:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829816.190295:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829816/real 1787829816]  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829816.190303:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829816.190304:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829816.190306:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000400:00000200:1.0:1787829816.190309:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000010:1.0:1787829816.190311:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000100:00000001:1.0:1787829816.190313:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829816.190317:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829816.190326:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190339:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829816.190341:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190343:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.190344:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829816.190347:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190351:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829816.190354:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190358:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829816.190360:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190361:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829816.190362:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190363:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829816.190364:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829816.190365:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829816.190367:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829816.190369:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829816.190371:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829816.190376:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382272:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829816.190378:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190379:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.190381:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed2d00 x1874671327382016/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190385:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829816.190387:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829816.190388:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829816.190390:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829816.190392:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829816.190393:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829816.190395:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829816.190396:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:1.0:1787829816.190399:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829816.190400:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829816.190402:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed2d00.
00000100:00000001:1.0:1787829816.190406:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829816.190407:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190409:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.190413:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed21c0 x1874671327382144/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190419:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829816.190421:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829816.190423:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829816.190425:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829816.190426:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829816.190428:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829816.190430:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829816.190431:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:1.0:1787829816.190434:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829816.190436:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829816.190437:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed21c0.
00000100:00000001:1.0:1787829816.190440:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829816.190441:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190443:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829816.190447:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0780 x1874671327382272/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829816.190451:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829816.190452:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829816.190453:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829816.190455:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829816.190456:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829816.190456:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829816.190458:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829816.190459:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:1.0:1787829816.190460:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829816.190461:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829816.190462:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0780.
00000100:00000001:1.0:1787829816.190463:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829816.190464:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190465:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829816.190468:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829816.190469:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829816.190470:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829816.190471:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829816.700786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829816.700791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829816.700793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829816.700797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829816.700799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829816.700803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829816.700805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829816.700807:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829816.700809:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829816.892764:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298636972, at 4298636992
00000100:00000040:3.0:1787829816.892764:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298636972, at 4298636992
00000100:00000010:2.0:1787829816.892778:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e639c0.
00000100:00000010:3.0:1787829816.892781:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63000.
00010000:00000001:0.0:1787829817.020785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829817.020792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829817.020799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829817.020801:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829817.020804:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829817.020805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829817.020856:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829817.020859:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829817.020862:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829817.724786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829817.724792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829817.724795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829817.724800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829817.724804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829817.724808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829817.724811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829817.724814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829817.724817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829818.044771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829818.044776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829818.044781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829818.044782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829818.044785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829818.044786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829818.044819:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829818.044823:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829818.044826:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829818.748786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829818.748793:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829818.748796:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829818.748801:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829818.748805:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829818.748808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829818.748811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829818.748815:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829818.748817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829819.068770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829819.068775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829819.068780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829819.068781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829819.068784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829819.068785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829819.068823:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829819.068825:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829819.068827:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829819.772785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829819.772791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829819.772793:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829819.772797:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829819.772800:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829819.772803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829819.772805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829819.772806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829819.772808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829820.092805:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829820.092811:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829820.092816:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829820.092817:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829820.092819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829820.092820:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829820.092886:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829820.092890:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829820.092894:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829820.796778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829820.796783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829820.796785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829820.796789:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829820.796792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829820.796795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829820.796797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829820.796798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829820.796800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829821.116781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829821.116786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829821.116790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829821.116792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829821.116794:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829821.116797:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829821.116799:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829821.116802:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829821.116804:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829821.116805:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829821.116807:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829821.116809:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829821.116852:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829821.116857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829821.116859:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829821.308798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829821.308806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829821.308808:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829821.308811:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829821.308814:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829821.308820:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3968
00000400:00000200:0.0:1787829821.308828:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829821.308833:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829821.308835:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829821.308837:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.308840:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829821.308842:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.308844:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829821.308846:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829821.308848:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829821.308850:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829821.308853:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829821.308854:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829821.308856:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829821.308857:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.308859:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829821.308860:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.308862:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829821.308864:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829821.308866:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829821.308868:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829821.308869:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829821.308872:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829821.308874:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829821.308875:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829821.308877:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829821.308886:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716da40.
00000020:00000040:0.0:1787829821.308890:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829821.308894:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829821.308895:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829821.308898:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829821.308901:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829821.308905:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829821.308914:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532800.
00000100:00000001:0.0:1787829821.308921:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829821.308930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829821.308942:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829821.308945:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829821.308949:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000226d20b5] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829821.308965:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829821.308968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829821.308971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829821.308972:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829821.308974:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829821.308977:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829821.308979:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3968
00000400:00000200:0.0:1787829821.308983:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829821.308986:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829821.308989:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829821.308992:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.308995:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829821.308998:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309001:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829821.309005:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829821.309006:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829821.309008:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787829821.309009:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787829821.309010:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829821.309015:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829821.309016:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787829821.309018:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829821.309019:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829821.309025:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829821.309029:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829821.309030:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829821.309036:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309043:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829821.309044:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829821.309048:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787829821.309049:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:1.0:1787829821.309051:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787829821.309055:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829821.309056:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829821.309062:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309062:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829821.309068:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787829821.309070:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327382400:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829821.309074:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829821.309076:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829821.309077:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829821.309080:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829821.309082:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309083:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829821.309087:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:1.0:1787829821.309089:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6000.
00000008:00100000:0.0:1787829821.309092:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829821.309093:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829821.309099:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309101:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829821.309104:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829821.309106:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630b58.
00000100:00000010:0.0:1787829821.309111:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716c000.
00000400:00000010:1.0:1787829821.309113:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98198.
00000020:00000040:0.0:1787829821.309116:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829821.309121:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:1.0:1787829821.309121:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382400, portal 25
02000000:00000001:0.0:1787829821.309125:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.309126:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829821.309128:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000001:0.0:1787829821.309131:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1787829821.309135:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829821.309136:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829821.309141:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.309146:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:0.0:1787829821.309150:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530c00.
00000100:00000040:1.0:1787829821.309150:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829821.309154:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98088.
00000100:00000001:0.0:1787829821.309157:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829821.309158:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327382400, offset 0
00000400:00000200:1.0:1787829821.309169:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00100000:0.0:1787829821.309171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829821.309178:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829821.309187:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:0.0:1787829821.309188:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829821.309193:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787829821.309194:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309198:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829821.309202:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829821.309204:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829821.309207:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40759:40759:256:4294967295] 192.168.203.116@tcp LPNI seq info [40759:40759:8:4294967295]
00000400:00000200:1.0:1787829821.309210:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829821.309214:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cdd0ad0f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829821.309224:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829821.309238:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309240:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787829821.309243:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955000.
00000100:00080000:0.0:1787829821.309245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1787829821.309251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:0.0:1787829821.309253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829821.309257:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.1:1787829821.309257:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:0.0:1787829821.309260:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829821.309265:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829821.309269:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3968
00000100:00000001:1.0:1787829821.309274:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829821.309275:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829821.309280:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829821.309284:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298642092
00000100:00000010:1.0:1787829821.309288:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d20.
00000400:00000200:0.0:1787829821.309290:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829821.309294:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829821.309295:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:0.0:1787829821.309300:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829821.309302:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298642092
00000100:00000001:0.0:1787829821.309305:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:1.0:1787829821.309306:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63480.
00000100:00000001:1.0:1787829821.309311:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000040:0.0:1787829821.309312:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309317:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829821.309320:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787829821.309321:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829821.309325:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000001:1.0:1787829821.309325:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829821.309329:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787829821.309331:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787829821.309333:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000001:1.0:1787829821.309334:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787829821.309339:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:0.0:1787829821.309344:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:1.0:1787829821.309344:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829821.309348:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000040:0.0:1787829821.309350:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829821.309352:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829821.309354:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829821.309355:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829821.309359:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829821.309362:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829821.309369:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829821.309370:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829821.309373:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829821.309375:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:1.0:1787829821.309376:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829821.309379:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787829821.309381:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
02000000:00000001:1.0:1787829821.309382:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829821.309389:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:1.0:1787829821.309389:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382528:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829821.309396:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787829821.309397:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309398:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829821.309402:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829821.309403:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309407:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787829821.309409:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:1.0:1787829821.309410:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829821.309415:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829821.309416:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309420:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829821.309421:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787829821.309425:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000008:00100000:0.0:1787829821.309426:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787829821.309430:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000008:00000001:0.0:1787829821.309431:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829821.309435:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382528, portal 10
00000100:00000001:0.0:1787829821.309436:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.309439:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829821.309441:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000010:0.0:1787829821.309444:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f0c0.
00000100:00000040:1.0:1787829821.309448:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829821.309450:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:0.0:1787829821.309456:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829821.309458:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787829821.309460:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829821.309464:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309466:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:1.0:1787829821.309468:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98e58.
02000000:00000001:0.0:1787829821.309470:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:1.0:1787829821.309472:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327382528, offset 0
00000400:00000200:1.0:1787829821.309475:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829821.309478:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:0.0:1787829821.309481:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309481:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829821.309488:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:0.0:1787829821.309490:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531800.
00000400:00000200:1.0:1787829821.309493:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829821.309498:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787829821.309501:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309503:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00100000:0.0:1787829821.309512:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829821.309514:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40760:40760:255:4294967295] 192.168.203.116@tcp LPNI seq info [40760:40760:7:4294967295]
00000400:00000200:1.0:1787829821.309527:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829821.309532:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829821.309538:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309543:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787829821.309548:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003ce953a3] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1787829821.309551:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829821.309557:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955200.
00000800:00000200:1.0:1787829821.309565:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829821.309567:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829821.309572:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:0.0:1787829821.309573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829821.309575:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829821.309582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000040:1.0:1787829821.309583:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000100:00000001:0.0:1787829821.309584:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829821.309589:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829821.309592:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829821.309595:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3968
00000400:00000200:1.0:1787829821.309600:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829821.309604:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829821.309606:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298641418
00000400:00000200:0.0:1787829821.309610:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829821.309612:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298642092
00000100:00080000:0.0:1787829821.309615:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829821.309617:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298641418
00000100:00000040:0.0:1787829821.309622:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829821.309623:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298642092
00000100:00000001:0.0:1787829821.309627:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829821.309628:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829821.309632:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829821.309634:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829821.309636:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829821.309639:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829821.309640:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787829821.309645:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787829821.309645:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829821.309648:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000020:00000040:0.0:1787829821.309649:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:1.0:1787829821.309653:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1787829821.309654:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829821.309703:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:0.0:1787829821.309706:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829821.309711:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:0.0:1787829821.309716:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309719:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829821.309722:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829821.309724:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829821.309727:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:1.0:1787829821.309731:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787829821.309734:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829821.309735:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787829821.309741:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829821.309745:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829821.309746:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829821.309750:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829821.309756:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829821.309761:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829821.309763:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:0.0:1787829821.309765:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829821.309766:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829821.309772:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309772:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829821.309775:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829821.309777:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309778:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829821.309783:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382656:192.168.203.116@tcp:8:kworker.0
00000100:00000010:0.0:1787829821.309784:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829821.309786:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829821.309794:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1787829821.309795:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829821.309796:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.309800:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787829821.309803:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829821.309806:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829821.309806:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:0.0:1787829821.309810:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829821.309812:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829821.309813:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829821.309817:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.309820:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:0.0:1787829821.309824:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532c00.
00000400:00000010:1.0:1787829821.309826:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630d68.
00000100:00000001:0.0:1787829821.309831:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829821.309833:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829821.309838:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382656, portal 4
00000100:00100000:0.0:1787829821.309842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829821.309842:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829821.309845:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000001:0.0:1787829821.309856:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787829821.309857:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829821.309861:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829821.309871:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc9f2c41] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829821.309872:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829821.309880:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829821.309886:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98aa0.
00000100:00000001:0.0:1787829821.309890:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829821.309891:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327382656, offset 0
00000100:00000040:0.0:1787829821.309899:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3979)
00000400:00000200:1.0:1787829821.309900:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829821.309907:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829821.309912:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829821.309916:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829821.309919:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829821.309921:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829821.309924:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829821.309928:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40761:40761:254:4294967295] 192.168.203.116@tcp LPNI seq info [40761:40761:6:4294967295]
00000400:00000200:1.0:1787829821.309932:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829821.309939:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829821.309944:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829821.309947:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955300.
00000800:00000200:1.0:1787829821.309951:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829821.309955:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829821.309958:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829821.309961:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829821.309964:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829821.309966:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298641426
00000100:00000001:1.0:1787829821.309969:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298642092
00000100:00000001:1.0:1787829821.309971:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298641426
00000100:00000001:1.0:1787829821.309973:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298642092
00000100:00000001:1.0:1787829821.309975:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829821.309977:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.309980:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.309982:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829821.309985:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829821.309987:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829821.309989:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829821.309990:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829821.309992:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829821.309996:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829821.310004:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829821.310005:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.310007:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829821.310009:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829821.310012:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829821.310014:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829821.310018:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829821.310019:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829821.310021:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829821.310022:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829821.310026:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829821.310028:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829821.310031:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829821.310034:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd946304d0.
00000400:00000010:1.0:1787829821.310038:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f985d8.
00000100:00000200:1.0:1787829821.310041:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382784, portal 4
00000100:00000001:1.0:1787829821.310043:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829821.310045:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829821.310050:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829821.310056:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829821.310058:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829821.310061:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f982a8.
00000100:00000200:1.0:1787829821.310063:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327382784, offset 0
00000400:00000200:1.0:1787829821.310067:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829821.310071:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829821.310074:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829821.310077:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829821.310080:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829821.310082:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829821.310084:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829821.310087:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40762:40762:253:4294967295] 192.168.203.116@tcp LPNI seq info [40762:40762:5:4294967295]
00000400:00000200:1.0:1787829821.310091:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829821.310097:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829821.310101:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829821.310105:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955900.
00000800:00000200:1.0:1787829821.310109:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829821.310113:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829821.310115:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829821.310118:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829821.310120:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829821.310122:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298641431
00000100:00000001:1.0:1787829821.310125:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298642092
00000100:00000001:1.0:1787829821.310127:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298641431
00000100:00000001:1.0:1787829821.310129:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298642092
00000100:00000001:1.0:1787829821.310131:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829821.310133:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.310136:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.310138:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829821.310140:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829821.310142:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787829821.310144:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829821.310145:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829821.310147:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829821.310148:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829821.820770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829821.820776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829821.820778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829821.820783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829821.820787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829821.820791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829821.820793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829821.820795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829821.820798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829822.012772:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298642092, at 4298642112
00000100:00000040:2.0:1787829822.012776:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298642092, at 4298642112
00000100:00000010:3.0:1787829822.012786:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63480.
00000100:00000010:2.0:1787829822.012787:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d20.
00010000:00000001:0.0:1787829822.140775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829822.140779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829822.140784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829822.140786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829822.140788:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829822.140790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829822.140809:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829822.140812:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829822.140815:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829822.844770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829822.844776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829822.844779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829822.844784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829822.844787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829822.844791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829822.844794:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829822.844796:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829822.844800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829823.164775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829823.164782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829823.164787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829823.164788:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829823.164790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829823.164791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829823.164868:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829823.164873:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829823.164877:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829823.868760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829823.868765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829823.868766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829823.868771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829823.868773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829823.868776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829823.868778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829823.868779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829823.868781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829824.188734:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829824.188739:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829824.188743:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829824.188743:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829824.188745:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829824.188746:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829824.188818:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829824.188823:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829824.188826:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829824.380901:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829824.380944:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829824.380953:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829824.380957:0:2335:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829824.380961:0:2335:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829824.380965:0:2335:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829824.380970:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829824.380974:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955000.
00000400:00000200:2.0:1787829824.380979:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829824.380983:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829824.380988:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98088
00000400:00000010:2.0:1787829824.380989:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98088.
00000100:00000001:2.0:1787829824.380993:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829824.381002:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829824.381027:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829824.381031:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829824.381039:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829824.381041:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829824.381050:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829824.381053:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955200.
00000400:00000200:2.0:1787829824.381057:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829824.381061:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829824.381063:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98e58
00000400:00000010:2.0:1787829824.381066:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98e58.
00000100:00000001:2.0:1787829824.381069:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787829824.381072:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000200:2.0:1787829824.381074:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381077:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829824.381083:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829824.381086:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:1.0:1787829824.381096:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829821/real 1787829824]  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829824.381096:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829824.381117:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829824.381121:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000001:1.0:1787829824.381124:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000800:00000100:2.0:1787829824.381128:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000400:00000001:1.0:1787829824.381129:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829824.381134:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630b58.
00000800:00000010:2.0:1787829824.381134:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955300.
00000400:00000200:1.0:1787829824.381139:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98198
00000400:00000200:2.0:1787829824.381140:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000010:1.0:1787829824.381143:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98198.
00000400:00000200:2.0:1787829824.381148:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1787829824.381149:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1787829824.381152:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98aa0
00000100:00000200:1.0:1787829824.381158:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787829824.381158:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98aa0.
00000100:00000001:2.0:1787829824.381164:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829824.381171:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829824.381173:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381196:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829824.381199:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829824.381199:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829824.381204:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381207:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829824.381207:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829824.381216:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829824.381220:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829824.381224:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829824.381230:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:2.0:1787829824.381234:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000100:00000040:1.0:1787829824.381239:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1787829824.381240:0:2335:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955900.
00000400:00000200:2.0:1787829824.381245:0:2335:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1787829824.381252:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:2.0:1787829824.381253:0:2335:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1787829824.381258:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381260:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381262:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1787829824.381265:0:2335:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f982a8
00000100:00000001:1.0:1787829824.381266:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829824.381269:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000010:2.0:1787829824.381270:0:2335:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f982a8.
00000100:00080000:1.0:1787829824.381275:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1787829824.381276:0:2335:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:1.0:1787829824.381284:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000200:2.0:1787829824.381284:0:2335:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381288:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829824.381296:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829824.381299:0:2335:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829824.381305:0:2335:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829824.381311:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327382400:192.168.203.116@tcp:250:kworker.0
00000100:00000001:2.0:1787829824.381319:0:2335:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829824.381321:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829824.381324:0:2335:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829824.381325:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829824.381331:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327382400/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381337:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829824.381340:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829824.381344:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6000.
02000000:00000001:1.0:1787829824.381348:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829824.381350:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829824.381352:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829824.381354:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829824.381357:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:1.0:1787829824.381360:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829824.381362:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829824.381364:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829824.381380:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829824.381381:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381384:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829824.381386:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829824.381387:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829824.381388:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829824.381392:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829821/real 1787829824]  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829824.381400:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829824.381402:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829824.381403:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd946304d0.
00000400:00000200:1.0:1787829824.381406:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f985d8
00000400:00000010:1.0:1787829824.381407:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f985d8.
00000100:00000001:1.0:1787829824.381410:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829824.381413:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829824.381419:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381425:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829824.381426:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381428:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381429:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829824.381432:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381438:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829824.381442:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381449:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829824.381451:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381453:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381455:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381456:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829824.381458:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829824.381460:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829824.381464:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829824.381466:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829824.381470:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829824.381477:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382784:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829824.381481:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829824.381485:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829821/real 1787829824]  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829824.381492:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829824.381493:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829824.381495:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630d68.
00000400:00000200:1.0:1787829824.381497:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829824.381499:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829824.381501:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829824.381504:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829824.381510:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381516:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829824.381517:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381518:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381519:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829824.381523:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381535:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829824.381539:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381545:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829824.381547:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381548:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381550:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381551:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829824.381553:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829824.381555:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829824.381557:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829824.381559:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829824.381563:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829824.381569:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382656:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829824.381571:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829824.381574:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829821/real 1787829824]  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829824.381578:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829824.381579:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829824.381581:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829824.381582:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829824.381583:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829824.381585:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829824.381587:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829824.381598:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381604:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829824.381605:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381607:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381608:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829824.381611:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381616:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829824.381619:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381625:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829824.381627:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381628:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829824.381629:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381631:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829824.381632:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829824.381640:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829824.381643:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829824.381645:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829824.381648:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829824.381695:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327382528:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829824.381701:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381703:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829824.381708:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327382784/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381718:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829824.381720:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829824.381722:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829824.381725:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829824.381726:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829824.381728:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829824.381729:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829824.381731:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:1.0:1787829824.381733:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829824.381734:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829824.381735:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829824.381737:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829824.381738:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381740:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829824.381743:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327382656/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381748:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829824.381749:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829824.381750:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829824.381752:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829824.381753:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829824.381754:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829824.381756:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829824.381757:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:1.0:1787829824.381758:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829824.381759:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829824.381761:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829824.381762:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829824.381764:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381765:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829824.381768:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327382528/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829824.381774:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829824.381775:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829824.381777:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829824.381778:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829824.381780:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829824.381781:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829824.381782:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829824.381783:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829824.381785:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829824.381786:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829824.381787:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829824.381790:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829824.381791:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381792:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829824.381799:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829824.381801:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829824.381810:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829824.381811:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829824.892743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829824.892751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829824.892753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829824.892759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829824.892762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829824.892766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829824.892768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829824.892771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829824.892773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212783:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829825.212791:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829825.212794:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829825.212796:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212800:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829825.212801:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212803:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212805:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829825.212808:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829825.212809:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829825.212810:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212811:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829825.212812:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212813:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212814:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829825.212817:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829825.212818:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829825.212837:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829825.212839:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829825.212841:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829825.916800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829825.916808:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829825.916812:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829825.916819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829825.916824:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829825.916830:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829825.916834:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829825.916836:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829825.916841:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829826.236751:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829826.236756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829826.236759:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829826.236761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829826.236764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829826.236767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829826.236817:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829826.236821:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829826.236824:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829826.428786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829826.428794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829826.428796:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829826.428799:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829826.428802:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829826.428809:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3974
00000400:00000200:1.0:1787829826.428818:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829826.428824:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829826.428828:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829826.428831:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.428834:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829826.428837:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.428840:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829826.428844:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829826.428847:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829826.428850:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829826.428853:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829826.428856:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829826.428859:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829826.428862:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.428863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829826.428865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.428867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829826.428870:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829826.428873:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829826.428876:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829826.428879:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829826.428882:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829826.428885:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.428887:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829826.428889:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829826.428896:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829826.428899:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829826.428903:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829826.428905:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.428907:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829826.428910:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829826.428913:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829826.428920:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829826.428927:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829826.428935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829826.428946:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829826.428949:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829826.428954:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829826.428977:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829826.428980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829826.428983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829826.428985:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829826.428987:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829826.428989:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829826.428992:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3974
00000400:00000200:1.0:1787829826.428997:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829826.429000:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829826.429003:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829826.429006:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829826.429009:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829826.429009:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829826.429012:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829826.429015:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829826.429016:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829826.429019:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787829826.429026:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:0.0:1787829826.429028:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829826.429029:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829826.429032:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829826.429035:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829826.429040:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829826.429043:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1787829826.429044:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829826.429046:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.429052:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:1.0:1787829826.429053:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429057:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829826.429059:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429061:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829826.429062:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829826.429066:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829826.429067:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829826.429073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00100000:0.0:1787829826.429075:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327382912:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829826.429078:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829826.429081:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829826.429086:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829826.429090:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.429094:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829826.429094:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829826.429100:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:1.0:1787829826.429101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829826.429105:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1787829826.429106:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.429110:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829826.429111:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829826.429116:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
02000000:00000010:0.0:1787829826.429119:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000020:00000040:1.0:1787829826.429119:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829826.429125:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829826.429126:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829826.429128:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.429132:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1787829826.429135:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829826.429136:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829826.429139:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829826.429142:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
02000000:00000010:1.0:1787829826.429146:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000400:00000010:0.0:1787829826.429150:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829826.429158:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829826.429160:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327382912, portal 25
00000100:00000001:0.0:1787829826.429166:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00100000:1.0:1787829826.429166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829826.429169:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000001:1.0:1787829826.429180:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1787829826.429181:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829826.429183:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829826.429193:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.429197:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829826.429204:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429207:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829826.429209:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00080000:1.0:1787829826.429211:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1787829826.429215:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327382912, offset 0
00000100:00080000:1.0:1787829826.429216:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829826.429220:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829826.429222:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829826.429224:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829826.429227:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829826.429230:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3974
00000400:00000200:1.0:1787829826.429236:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829826.429239:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829826.429241:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829826.429244:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429246:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829826.429248:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829826.429251:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429254:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829826.429257:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829826.429258:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000040:1.0:1787829826.429260:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829826.429263:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829826.429266:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829826.429267:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1787829826.429270:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829826.429274:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:1.0:1787829826.429275:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829826.429282:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1787829826.429283:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429288:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829826.429289:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:1.0:1787829826.429293:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429298:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829826.429299:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40763:40763:256:4294967295] 192.168.203.116@tcp LPNI seq info [40763:40763:8:4294967295]
00000020:00000040:1.0:1787829826.429304:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829826.429308:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829826.429311:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429315:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829826.429317:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829826.429321:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829826.429324:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1787829826.429326:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.429329:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829826.429335:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.429341:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000008:00100000:1.0:1787829826.429343:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829826.429348:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829826.429350:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829826.429352:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829826.429356:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000100:00000010:1.0:1787829826.429358:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829826.429363:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.1:1787829826.429364:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:1.0:1787829826.429367:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829826.429369:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.429371:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829826.429372:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829826.429374:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829826.429378:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829826.429383:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829826.429384:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829826.429393:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00100000:1.0:1787829826.429393:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1787829826.429396:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829826.429403:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787829826.429405:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829826.429410:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829826.429411:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000040:1.0:1787829826.429418:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829826.429419:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829826.429423:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829826.429429:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787829826.429434:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829826.429437:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829826.429438:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829826.429447:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829826.429451:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829826.429454:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829826.429456:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829826.429458:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:0.0:1787829826.429463:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829826.429466:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3974
00000400:00000200:1.0:1787829826.429475:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829826.429479:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829826.429481:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829826.429483:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:1.0:1787829826.429485:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829826.429489:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829826.429491:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.429492:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829826.429497:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829826.429501:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829826.429502:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829826.429505:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298647212
00000020:00000001:1.0:1787829826.429507:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829826.429510:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829826.429513:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:0.0:1787829826.429514:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccea0.
00000020:00000040:1.0:1787829826.429516:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829826.429520:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829826.429523:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829826.429524:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787829826.429527:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829826.429531:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298647212
00000100:00000001:1.0:1787829826.429532:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829826.429536:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:0.0:1787829826.429539:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc780.
00000100:00000001:1.0:1787829826.429540:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829826.429546:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829826.429547:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:1.0:1787829826.429550:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829826.429554:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829826.429557:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829826.429558:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829826.429560:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829826.429564:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829826.429565:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.429570:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829826.429570:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829826.429574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1787829826.429578:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826]  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829826.429579:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829826.429583:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829826.429589:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829826.429594:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000001:0.0:1787829826.429595:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:1.0:1787829826.429597:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:0.0:1787829826.429598:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787829826.429600:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829826.429602:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000001:1.0:1787829826.429604:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829826.429607:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829826.429609:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
02000000:00000001:1.0:1787829826.429610:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829826.429613:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
02000000:00000010:1.0:1787829826.429616:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:0.0:1787829826.429618:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829826.429623:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829826.429624:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829826.429632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829826.429641:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829826.429642:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829826.429686:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829826.429689:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.429692:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.429694:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829826.429699:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829826.429700:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.429710:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829826.429711:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829826.429721:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829826.429725:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829826.429733:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3984)
00000100:00000001:0.0:1787829826.429735:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829826.429739:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.429741:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.429743:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.429745:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829826.429747:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829826.429750:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829826.429754:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829826.429757:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829826.429761:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829826.429768:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327382912:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829826.429772:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.429774:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.429777:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327382912/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.429782:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829826.429785:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829826.429787:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829826.429790:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829826.429792:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829826.429794:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829826.429797:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829826.429799:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829826.429803:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829826.429805:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829826.429807:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829826.429810:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829826.429812:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.429814:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.429821:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829826.429823:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.429825:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829826.429827:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829826.429828:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829826.429832:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.429838:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829826.429839:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.429841:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829826.429843:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829826.429845:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829826.429848:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383040:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829826.429851:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829826.429853:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829826.429854:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.429856:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829826.429858:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829826.429860:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829826.429863:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829826.429865:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829826.429870:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829826.429873:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383040, portal 10
00000100:00000001:0.0:1787829826.429875:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829826.429876:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829826.429883:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.429888:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829826.429890:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829826.429893:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829826.429895:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327383040, offset 0
00000400:00000200:0.0:1787829826.429899:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829826.429903:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829826.429906:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829826.429909:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829826.429918:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829826.429920:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829826.429923:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829826.429925:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40764:40764:256:4294967295] 192.168.203.116@tcp LPNI seq info [40764:40764:8:4294967295]
00000400:00000200:0.0:1787829826.429929:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829826.429935:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829826.429939:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.429942:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829826.429946:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829826.429949:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829826.429953:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829826.429956:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829826.429974:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.429976:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829826.429979:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829826.429982:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829826.429986:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829826.429988:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829826.429990:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829826.429994:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430006:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.430009:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430014:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430015:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829826.430018:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829826.430022:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829826.430025:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829826.430027:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298646539
00000100:00000001:0.0:1787829826.430029:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298647212
00000100:00000001:0.0:1787829826.430031:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298646539
00000100:00000001:0.0:1787829826.430033:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298647212
00000100:00000001:0.0:1787829826.430035:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829826.430037:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430038:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829826.430041:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826]  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829826.430048:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829826.430050:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829826.430051:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829826.430054:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829826.430055:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829826.430057:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829826.430059:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829826.430066:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430071:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829826.430073:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430074:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.430075:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829826.430078:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430083:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829826.430086:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430091:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829826.430093:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430094:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.430096:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430097:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829826.430099:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829826.430101:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829826.430103:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829826.430105:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829826.430108:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829826.430115:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383040:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829826.430118:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829826.430121:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430126:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829826.430127:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.430129:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829826.430131:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829826.430132:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829826.430136:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829826.430138:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829826.430140:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829826.430142:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.430144:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829826.430147:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829826.430149:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829826.430152:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829826.430154:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829826.430157:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829826.430160:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383168, portal 4
00000100:00000001:0.0:1787829826.430161:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829826.430162:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829826.430167:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430173:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829826.430176:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829826.430178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829826.430181:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327383168, offset 0
00000400:00000200:0.0:1787829826.430184:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829826.430187:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829826.430191:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829826.430193:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829826.430196:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829826.430198:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829826.430200:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829826.430202:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40765:40765:256:4294967295] 192.168.203.116@tcp LPNI seq info [40765:40765:8:4294967295]
00000400:00000200:0.0:1787829826.430205:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829826.430211:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829826.430215:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.430218:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829826.430222:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829826.430225:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829826.430228:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829826.430231:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829826.430234:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.430236:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829826.430238:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829826.430241:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829826.430249:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829826.430250:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829826.430253:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829826.430256:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430264:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.430266:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430272:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430273:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829826.430277:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829826.430279:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829826.430282:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829826.430284:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298646546
00000100:00000001:0.0:1787829826.430287:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298647212
00000100:00000001:0.0:1787829826.430289:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298646546
00000100:00000001:0.0:1787829826.430291:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298647212
00000100:00000001:0.0:1787829826.430293:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829826.430295:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430297:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829826.430300:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826]  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829826.430308:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829826.430310:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829826.430311:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829826.430313:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829826.430314:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829826.430316:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829826.430318:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829826.430323:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430328:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829826.430329:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430330:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.430332:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829826.430335:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430341:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829826.430344:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430350:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829826.430352:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430354:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.430355:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430357:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829826.430358:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829826.430361:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829826.430364:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829826.430366:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829826.430370:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829826.430384:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829826.430388:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829826.430391:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430396:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829826.430398:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.430400:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829826.430402:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829826.430404:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829826.430407:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383296:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829826.430410:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829826.430412:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829826.430413:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829826.430415:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829826.430418:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829826.430420:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829826.430423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829826.430426:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829826.430429:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829826.430431:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383296, portal 4
00000100:00000001:0.0:1787829826.430433:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829826.430434:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:0.0:1787829826.430439:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430445:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829826.430447:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829826.430449:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829826.430451:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327383296, offset 0
00000400:00000200:0.0:1787829826.430455:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829826.430459:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829826.430463:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829826.430466:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829826.430468:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829826.430470:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829826.430472:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829826.430475:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40766:40766:256:4294967295] 192.168.203.116@tcp LPNI seq info [40766:40766:8:4294967295]
00000400:00000200:0.0:1787829826.430478:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829826.430484:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829826.430488:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.430490:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829826.430494:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829826.430496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829826.430500:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829826.430502:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829826.430506:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829826.430507:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829826.430510:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829826.430513:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829826.430516:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829826.430517:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829826.430519:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829826.430522:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430529:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.430532:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430537:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430539:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829826.430541:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829826.430544:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829826.430547:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829826.430549:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298646551
00000100:00000001:0.0:1787829826.430551:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298647212
00000100:00000001:0.0:1787829826.430553:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298646551
00000100:00000001:0.0:1787829826.430555:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298647212
00000100:00000001:0.0:1787829826.430557:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829826.430559:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430560:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829826.430563:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826]  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829826.430570:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829826.430571:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829826.430572:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829826.430574:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829826.430576:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829826.430577:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829826.430580:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829826.430587:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430593:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829826.430594:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430595:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.430596:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829826.430600:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430605:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829826.430609:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430615:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829826.430618:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430620:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829826.430621:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430622:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829826.430624:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829826.430626:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829826.430629:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829826.430632:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829826.430635:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829826.430643:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383296:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829826.430681:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430684:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.430688:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327383040/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430694:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829826.430696:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829826.430697:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829826.430700:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829826.430701:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829826.430703:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829826.430705:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829826.430706:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829826.430709:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829826.430710:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829826.430712:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829826.430714:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829826.430715:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430717:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.430720:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327383168/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430731:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829826.430732:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829826.430733:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829826.430735:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829826.430737:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829826.430738:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829826.430739:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829826.430741:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:0.0:1787829826.430743:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829826.430744:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829826.430745:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829826.430747:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829826.430748:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430750:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829826.430752:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327383296/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829826.430759:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829826.430760:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829826.430761:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829826.430763:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829826.430764:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829826.430765:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829826.430767:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829826.430768:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829826.430770:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829826.430771:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829826.430772:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829826.430774:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829826.430775:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430777:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.430781:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829826.430783:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.430784:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829826.430786:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829826.940747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829826.940752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829826.940753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829826.940756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829826.940759:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829826.940761:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.940763:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829826.940765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829826.940767:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829827.132750:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298647212, at 4298647232
00000100:00000040:3.0:1787829827.132750:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298647212, at 4298647232
00000100:00000010:3.0:1787829827.132762:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00000100:00000010:2.0:1787829827.132767:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00010000:00000001:0.0:1787829827.260745:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829827.260751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829827.260756:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829827.260758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829827.260761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829827.260762:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829827.260793:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829827.260797:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829827.260799:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829827.964759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829827.964764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829827.964765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829827.964769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829827.964771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829827.964773:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829827.964775:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829827.964776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829827.964778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829828.284774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829828.284781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829828.284785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829828.284786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829828.284788:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829828.284789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829828.284808:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829828.284811:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829828.284813:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829828.988730:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829828.988738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829828.988741:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829828.988747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829828.988751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829828.988755:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829828.988758:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829828.988762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829828.988765:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829829.308772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829829.308779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829829.308784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829829.308785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829829.308788:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829829.308789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829829.308862:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829829.308867:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829829.308870:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829830.012800:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829830.012813:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829830.012816:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829830.012821:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829830.012824:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829830.012827:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829830.012829:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829830.012831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829830.012833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829830.332793:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829830.332810:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829830.332817:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829830.332819:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829830.332823:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829830.332827:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829830.332830:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829830.332834:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829830.332837:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829830.332840:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829830.332842:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829830.332844:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829830.332890:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829830.332894:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829830.332898:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829831.036713:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829831.036719:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829831.036720:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829831.036724:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829831.036726:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829831.036729:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829831.036731:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829831.036732:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829831.036735:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829831.356747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829831.356751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829831.356755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829831.356756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829831.356758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829831.356760:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829831.356774:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829831.356775:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829831.356777:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829831.548798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829831.548807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829831.548810:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829831.548814:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829831.548818:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829831.548826:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3979
00000400:00000200:0.0:1787829831.548834:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829831.548839:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829831.548843:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829831.548845:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.548848:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829831.548851:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.548853:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829831.548857:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829831.548859:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829831.548862:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829831.548865:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829831.548867:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829831.548869:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829831.548872:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.548875:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829831.548877:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.548879:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829831.548882:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829831.548885:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829831.548887:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829831.548890:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829831.548893:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829831.548896:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829831.548899:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829831.548901:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829831.548910:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716de00.
00000020:00000040:0.0:1787829831.548913:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829831.548916:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829831.548918:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829831.548921:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829831.548924:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829831.548927:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829831.548935:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531800.
00000100:00000001:0.0:1787829831.548941:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829831.548950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829831.548960:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829831.548964:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829831.548969:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c17e5d1d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829831.548991:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829831.548995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829831.548998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829831.548999:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829831.549001:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829831.549004:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829831.549007:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3979
00000400:00000200:0.0:1787829831.549012:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829831.549015:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787829831.549016:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829831.549020:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787829831.549021:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829831.549023:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829831.549027:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829831.549031:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829831.549032:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829831.549038:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549043:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829831.549044:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1787829831.549047:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787829831.549047:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787829831.549051:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
02000000:00000001:1.0:1787829831.549052:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1787829831.549057:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:1.0:1787829831.549057:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829831.549061:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787829831.549062:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00100000:1.0:1787829831.549065:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327383424:192.168.203.116@tcp:250:kworker.0
00000020:00000001:0.0:1787829831.549066:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829831.549071:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1787829831.549072:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:1.0:1787829831.549077:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787829831.549078:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549081:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787829831.549081:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829831.549086:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:1.0:1787829831.549087:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829831.549091:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829831.549094:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000010:1.0:1787829831.549094:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
00000100:00080000:0.0:1787829831.549099:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549101:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.549104:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829831.549105:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829831.549112:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787829831.549113:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:0.0:1787829831.549117:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829831.549121:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787829831.549121:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000008:00100000:0.0:1787829831.549127:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787829831.549129:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000008:00000001:0.0:1787829831.549132:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829831.549136:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829831.549140:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716ed00.
00000100:00000200:1.0:1787829831.549141:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383424, portal 25
00000020:00000040:0.0:1787829831.549143:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:1.0:1787829831.549144:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829831.549147:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
02000000:00000001:0.0:1787829831.549149:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829831.549152:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.549154:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829831.549156:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829831.549158:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829831.549161:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829831.549167:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530800.
00000100:00000001:1.0:1787829831.549168:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787829831.549173:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.549174:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829831.549180:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00100000:0.0:1787829831.549183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787829831.549185:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327383424, offset 0
00000100:00000001:0.0:1787829831.549196:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829831.549196:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549201:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829831.549206:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829831.549209:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008f0359f0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829831.549213:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829831.549220:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829831.549225:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:0.0:1787829831.549227:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829831.549229:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1787829831.549232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829831.549233:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:0.0:1787829831.549239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1787829831.549239:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40767:40767:256:4294967295] 192.168.203.116@tcp LPNI seq info [40767:40767:8:4294967295]
00000100:00000001:0.0:1787829831.549244:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787829831.549246:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829831.549247:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829831.549252:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829831.549258:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3979
00000400:00000200:0.0:1787829831.549263:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.549264:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787829831.549269:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000200:1.0:1787829831.549273:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00080000:0.0:1787829831.549275:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000010:1.0:1787829831.549277:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00000040:0.0:1787829831.549281:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549285:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787829831.549286:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:0.0:1787829831.549290:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829831.549293:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1787829831.549294:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829831.549300:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829831.549302:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:1.1:1787829831.549305:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:0.0:1787829831.549306:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829831.549310:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:1.1:1787829831.549312:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787829831.549313:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:0.0:1787829831.549319:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000100:1.0:1787829831.549321:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000040:0.0:1787829831.549326:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000010:1.0:1787829831.549326:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000001:0.0:1787829831.549330:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829831.549334:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1787829831.549335:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549340:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829831.549340:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1787829831.549346:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:1.0:1787829831.549346:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829831.549350:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00080000:0.0:1787829831.549354:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829831.549355:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829831.549360:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829831.549361:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829831.549363:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829831.549367:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829831.549370:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787829831.549372:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.549374:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829831.549376:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829831.549380:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829831.549382:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829831.549385:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549385:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829831.549390:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549391:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829831.549394:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f480.
00000400:00000200:1.0:1787829831.549396:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:0.0:1787829831.549400:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:1.0:1787829831.549401:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298652332
02000000:00000001:0.0:1787829831.549406:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829831.549409:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829831.549409:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63b40.
00000100:00000001:0.0:1787829831.549414:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829831.549416:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829831.549418:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549420:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000010:0.0:1787829831.549424:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533800.
00000100:00000001:1.0:1787829831.549427:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298652332
00000100:00000001:0.0:1787829831.549428:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829831.549433:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63d80.
00000100:00100000:0.0:1787829831.549437:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829831.549439:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829831.549448:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829831.549451:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549453:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829831.549456:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549458:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1787829831.549462:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84d4029] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829831.549463:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831]  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829831.549493:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787829831.549494:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:0.0:1787829831.549497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1787829831.549498:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829831.549501:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000100:00080000:0.0:1787829831.549503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787829831.549506:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000100:00000001:0.0:1787829831.549507:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829831.549510:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1787829831.549511:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:0.0:1787829831.549514:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787829831.549516:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:0.0:1787829831.549520:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3979
00000100:00000200:1.0:1787829831.549522:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829831.549528:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829831.549536:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000200:1.0:1787829831.549536:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829831.549541:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829831.549544:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829831.549547:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829831.549549:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829831.549550:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.549553:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829831.549555:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1787829831.549558:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829831.549561:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549568:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829831.549569:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787829831.549572:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829831.549574:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:0.0:1787829831.549578:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787829831.549579:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829831.549581:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:0.0:1787829831.549586:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829831.549590:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:1.0:1787829831.549596:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787829831.549597:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829831.549602:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829831.549603:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829831.549607:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829831.549608:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.549621:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829831.549624:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829831.549624:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829831.549626:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:0.0:1787829831.549628:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829831.549632:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829831.549634:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829831.549639:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829831.549640:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
02000000:00000001:0.0:1787829831.549642:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829831.549644:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:0.0:1787829831.549647:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829831.549652:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.549652:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829831.549685:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327383424:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829831.549691:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.549694:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787829831.549697:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:1.0:1787829831.549698:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327383424/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787829831.549701:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829831.549707:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829831.549711:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549713:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000010:0.0:1787829831.549716:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716d2c0.
02000000:00000001:1.0:1787829831.549717:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000040:0.0:1787829831.549721:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000010:1.0:1787829831.549722:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829831.549727:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829831.549728:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:0.0:1787829831.549730:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829831.549732:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000001:0.0:1787829831.549734:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000020:00000040:1.0:1787829831.549735:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829831.549738:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000020:00000001:1.0:1787829831.549740:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000001:0.0:1787829831.549742:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829831.549744:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000010:0.0:1787829831.549748:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829831.549750:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829831.549754:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:0.0:1787829831.549755:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829831.549757:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829831.549762:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00100000:0.0:1787829831.549763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829831.549765:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.549770:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829831.549778:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829831.549779:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829831.549781:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829831.549782:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549784:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829831.549787:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829831.549789:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829831.549792:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c4c777e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829831.549794:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.549817:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829831.549819:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.549821:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787829831.549822:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.549823:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829831.549827:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829831.549829:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383552:192.168.203.116@tcp:38:kworker.0
00000100:00000040:0.0:1787829831.549830:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3989)
00000100:00000001:1.0:1787829831.549833:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829831.549835:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829831.549836:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.549838:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829831.549841:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829831.549842:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829831.549845:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829831.549847:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829831.549850:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829831.549852:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383552, portal 10
00000100:00000001:1.0:1787829831.549853:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829831.549854:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829831.549859:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.549862:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829831.549871:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829831.549873:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829831.549876:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327383552, offset 0
00000400:00000200:1.0:1787829831.549879:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829831.549883:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.549886:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829831.549889:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829831.549892:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829831.549894:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829831.549896:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829831.549899:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40768:40768:256:4294967295] 192.168.203.116@tcp LPNI seq info [40768:40768:8:4294967295]
00000400:00000200:1.0:1787829831.549903:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.549910:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829831.549914:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829831.549916:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829831.549919:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829831.549927:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829831.549932:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829831.549934:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787829831.549938:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829831.549939:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829831.549943:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829831.549946:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829831.549949:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829831.549950:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829831.549953:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829831.549956:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.549964:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829831.549967:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.549970:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549971:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829831.549974:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.549976:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829831.549978:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829831.549979:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298651659
00000100:00000001:1.0:1787829831.549981:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298652332
00000100:00000001:1.0:1787829831.549982:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298651659
00000100:00000001:1.0:1787829831.549984:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298652332
00000100:00000001:1.0:1787829831.549985:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829831.549986:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.549987:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829831.549989:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831]  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829831.549994:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829831.549995:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829831.549996:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829831.549998:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829831.549999:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829831.550000:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829831.550002:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829831.550007:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550010:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829831.550011:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550012:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.550013:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829831.550015:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550018:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829831.550020:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550024:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829831.550025:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550026:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.550027:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550028:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829831.550029:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829831.550030:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829831.550032:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829831.550033:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829831.550035:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829831.550039:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383552:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829831.550041:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829831.550043:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550051:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829831.550052:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.550053:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829831.550054:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829831.550055:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829831.550057:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383680:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829831.550059:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829831.550060:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829831.550061:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.550062:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829831.550064:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829831.550065:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829831.550067:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829831.550069:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829831.550072:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829831.550074:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383680, portal 4
00000100:00000001:1.0:1787829831.550076:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829831.550077:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829831.550082:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550088:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829831.550090:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829831.550092:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829831.550094:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327383680, offset 0
00000400:00000200:1.0:1787829831.550098:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829831.550102:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.550105:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829831.550108:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829831.550111:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829831.550113:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829831.550115:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829831.550118:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40769:40769:256:4294967295] 192.168.203.116@tcp LPNI seq info [40769:40769:8:4294967295]
00000400:00000200:1.0:1787829831.550120:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.550125:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829831.550127:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829831.550129:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829831.550132:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829831.550133:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829831.550136:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829831.550137:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787829831.550139:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829831.550141:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829831.550142:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829831.550144:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829831.550146:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829831.550147:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829831.550149:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829831.550151:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550156:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829831.550159:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550165:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.550166:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829831.550169:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.550171:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829831.550174:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829831.550176:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298651666
00000100:00000001:1.0:1787829831.550178:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298652332
00000100:00000001:1.0:1787829831.550180:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298651666
00000100:00000001:1.0:1787829831.550181:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298652332
00000100:00000001:1.0:1787829831.550184:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829831.550186:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.550188:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829831.550191:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831]  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829831.550199:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829831.550199:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829831.550201:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829831.550202:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829831.550203:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829831.550204:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829831.550206:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829831.550211:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550214:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829831.550215:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550216:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.550217:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829831.550219:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550223:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829831.550225:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550229:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829831.550230:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550231:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.550232:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550232:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829831.550233:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829831.550235:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829831.550237:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829831.550238:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829831.550241:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829831.550245:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383680:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829831.550247:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550248:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829831.550249:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327383552/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550253:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829831.550254:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829831.550255:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829831.550256:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829831.550257:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829831.550258:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829831.550259:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829831.550260:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829831.550261:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829831.550262:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829831.550263:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829831.550265:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829831.550265:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550266:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829831.550268:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327383680/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550271:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829831.550272:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829831.550272:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829831.550273:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829831.550274:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829831.550275:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829831.550276:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829831.550277:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829831.550278:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829831.550279:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829831.550279:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829831.550280:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829831.550281:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550282:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550284:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829831.550285:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.550286:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829831.550287:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829831.550289:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829831.550292:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550298:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829831.550299:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.550301:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829831.550302:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829831.550304:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829831.550307:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383808:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829831.550310:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829831.550312:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829831.550313:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829831.550315:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829831.550318:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829831.550320:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829831.550323:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829831.550325:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829831.550328:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829831.550330:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383808, portal 4
00000100:00000001:1.0:1787829831.550332:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829831.550333:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829831.550337:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550344:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829831.550346:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829831.550347:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829831.550349:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327383808, offset 0
00000400:00000200:1.0:1787829831.550351:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829831.550353:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.550355:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829831.550357:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829831.550359:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829831.550360:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829831.550362:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829831.550363:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40770:40770:256:4294967295] 192.168.203.116@tcp LPNI seq info [40770:40770:8:4294967295]
00000400:00000200:1.0:1787829831.550366:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829831.550370:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829831.550372:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829831.550374:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829831.550376:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829831.550378:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787829831.550380:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829831.550382:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787829831.550384:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829831.550385:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829831.550387:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829831.550389:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829831.550390:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829831.550391:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829831.550393:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829831.550395:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550399:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829831.550401:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550412:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.550413:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829831.550416:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829831.550419:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829831.550421:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829831.550423:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298651671
00000100:00000001:1.0:1787829831.550425:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298652332
00000100:00000001:1.0:1787829831.550427:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298651671
00000100:00000001:1.0:1787829831.550429:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298652332
00000100:00000001:1.0:1787829831.550432:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829831.550434:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.550435:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829831.550439:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831]  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829831.550447:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829831.550448:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829831.550449:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829831.550450:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829831.550451:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829831.550453:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829831.550455:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829831.550459:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550463:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829831.550463:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550464:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.550465:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829831.550468:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550475:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829831.550479:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550484:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829831.550487:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550488:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829831.550489:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550491:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829831.550492:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829831.550495:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829831.550498:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829831.550500:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829831.550504:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829831.550509:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327383808:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829831.550512:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550513:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829831.550515:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327383808/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829831.550518:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829831.550519:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829831.550520:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829831.550521:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829831.550522:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829831.550523:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829831.550524:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829831.550525:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533c00.
02000000:00000001:1.0:1787829831.550526:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829831.550528:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829831.550528:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829831.550530:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829831.550531:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550532:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829831.550534:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829831.550535:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829831.550536:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829831.550537:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829832.060756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829832.060762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829832.060764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829832.060768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829832.060771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829832.060778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829832.060781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829832.060783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829832.060786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829832.252764:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298652332, at 4298652352
00000100:00000040:3.0:1787829832.252764:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298652332, at 4298652352
00000100:00000010:2.0:1787829832.252779:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63b40.
00000100:00000010:3.0:1787829832.252780:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63d80.
00010000:00000001:0.0:1787829832.380744:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829832.380749:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829832.380753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829832.380754:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829832.380757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829832.380759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829832.380798:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829832.380801:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829832.380803:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829833.084746:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829833.084752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829833.084753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829833.084757:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829833.084760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829833.084762:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829833.084764:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829833.084766:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829833.084768:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829833.404730:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829833.404735:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829833.404739:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829833.404740:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829833.404743:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829833.404743:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829833.404774:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829833.404777:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829833.404779:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829834.108744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829834.108749:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829834.108750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829834.108753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829834.108756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829834.108758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829834.108760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829834.108762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829834.108763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829834.428775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829834.428780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829834.428783:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829834.428784:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829834.428791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829834.428792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829834.428806:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829834.428808:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829834.428809:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829835.132749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829835.132755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829835.132756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829835.132760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829835.132762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829835.132765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829835.132767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829835.132768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829835.132773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452736:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829835.452742:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829835.452743:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829835.452746:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452749:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829835.452750:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452752:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452753:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829835.452756:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829835.452757:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829835.452758:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452758:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829835.452759:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452760:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452761:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829835.452763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829835.452764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829835.452804:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829835.452807:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829835.452809:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829836.156751:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829836.156757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829836.156758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829836.156762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829836.156765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829836.156768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829836.156769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829836.156771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829836.156773:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829836.476756:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829836.476760:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829836.476763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829836.476764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829836.476767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829836.476768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829836.476792:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829836.476796:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829836.476800:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829836.668784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829836.668794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829836.668797:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829836.668801:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829836.668805:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829836.668813:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3984
00000400:00000200:0.0:1787829836.668822:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829836.668829:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829836.668832:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829836.668836:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.668839:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829836.668841:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.668843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829836.668847:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829836.668849:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829836.668852:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829836.668856:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829836.668858:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829836.668861:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829836.668864:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.668866:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829836.668868:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.668870:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829836.668873:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829836.668876:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829836.668879:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829836.668882:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829836.668885:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829836.668889:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.668891:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829836.668894:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829836.668902:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716d2c0.
00000020:00000040:0.0:1787829836.668906:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829836.668909:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829836.668911:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.668913:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829836.668916:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829836.668919:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829836.668927:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533400.
00000100:00000001:0.0:1787829836.668933:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829836.668942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829836.668952:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829836.668956:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829836.668960:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c4c777e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829836.668980:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829836.668983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829836.668986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829836.668988:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829836.668989:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829836.668992:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829836.668995:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3984
00000400:00000200:0.0:1787829836.668998:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829836.669002:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829836.669004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829836.669007:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669010:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829836.669013:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669015:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829836.669015:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829836.669019:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787829836.669020:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787829836.669022:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787829836.669023:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829836.669028:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787829836.669029:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829836.669031:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829836.669036:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:0.0:1787829836.669041:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829836.669042:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829836.669045:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829836.669047:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669052:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787829836.669052:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787829836.669056:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669059:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829836.669061:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:1.0:1787829836.669062:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787829836.669065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829836.669067:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669070:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829836.669073:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787829836.669075:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327383936:192.168.203.116@tcp:250:kworker.0
02000000:00000001:0.0:1787829836.669080:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829836.669080:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829836.669086:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829836.669088:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.669090:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787829836.669094:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829836.669094:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829836.669098:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669099:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787829836.669104:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:0.0:1787829836.669105:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787829836.669108:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000008:00000001:0.0:1787829836.669110:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669114:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829836.669114:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000010:0.0:1787829836.669121:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f480.
00000100:00000200:1.0:1787829836.669121:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327383936, portal 25
00000100:00000001:1.0:1787829836.669125:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000040:0.0:1787829836.669127:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:1.0:1787829836.669127:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
02000000:00000001:0.0:1787829836.669133:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1787829836.669136:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829836.669137:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669141:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829836.669144:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829836.669147:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669148:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829836.669152:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000010:0.0:1787829836.669154:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531000.
00000400:00000010:1.0:1787829836.669155:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829836.669160:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327383936, offset 0
00000100:00000001:0.0:1787829836.669161:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829836.669165:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00100000:0.0:1787829836.669169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829836.669173:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.669181:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:0.0:1787829836.669183:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829836.669186:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787829836.669187:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829836.669191:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000040:0.0:1787829836.669195:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84d4029] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829836.669202:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787829836.669206:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829836.669207:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:0.0:1787829836.669210:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829836.669216:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1787829836.669216:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40771:40771:256:4294967295] 192.168.203.116@tcp LPNI seq info [40771:40771:8:4294967295]
00000100:00000001:0.0:1787829836.669219:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829836.669222:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829836.669226:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829836.669227:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829836.669231:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3984
00000400:00000200:0.0:1787829836.669238:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.669244:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787829836.669245:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829836.669251:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1787829836.669256:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:0.0:1787829836.669258:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000010:1.0:1787829836.669261:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00000001:0.0:1787829836.669262:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829836.669266:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829836.669270:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829836.669271:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829836.669276:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:1.0:1787829836.669276:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000020:00000040:0.0:1787829836.669278:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:0.0:1787829836.669281:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.1:1787829836.669283:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:0.0:1787829836.669285:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:1.1:1787829836.669287:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787829836.669288:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:0.0:1787829836.669293:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000100:1.0:1787829836.669293:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.669295:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000040:0.0:1787829836.669299:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829836.669300:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787829836.669303:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829836.669306:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1787829836.669309:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829836.669311:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000100:00000001:0.0:1787829836.669312:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:1.0:1787829836.669314:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000020:00000040:0.0:1787829836.669317:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:1.0:1787829836.669319:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1787829836.669324:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000200:1.0:1787829836.669325:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829836.669329:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829836.669335:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829836.669337:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787829836.669340:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787829836.669341:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829836.669346:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669351:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829836.669352:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669354:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000008:00100000:0.0:1787829836.669355:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829836.669359:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829836.669360:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669364:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829836.669364:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829836.669369:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298657452
00000100:00000010:0.0:1787829836.669370:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716ed00.
00000100:00000010:1.0:1787829836.669372:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63a20.
00000020:00000040:0.0:1787829836.669374:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:0.0:1787829836.669379:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829836.669379:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:0.0:1787829836.669381:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669384:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298657452
00000100:00000001:0.0:1787829836.669385:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:1.0:1787829836.669389:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63f00.
02000000:00000001:0.0:1787829836.669390:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829836.669394:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669394:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000010:0.0:1787829836.669401:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532400.
00000100:00000001:1.0:1787829836.669404:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829836.669405:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669407:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669411:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829836.669416:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1787829836.669416:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836]  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829836.669443:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787829836.669444:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1787829836.669446:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787829836.669449:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829836.669449:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829836.669454:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829836.669456:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000040:0.0:1787829836.669458:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008f0359f0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669459:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829836.669463:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829836.669481:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829836.669483:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829836.669485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829836.669494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829836.669495:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829836.669499:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787829836.669499:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787829836.669504:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829836.669504:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.669506:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829836.669510:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:1.0:1787829836.669511:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829836.669517:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3984
00000100:00000001:1.0:1787829836.669520:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787829836.669525:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829836.669526:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829836.669533:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829836.669534:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829836.669537:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787829836.669538:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.669540:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.669543:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829836.669545:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829836.669546:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829836.669548:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829836.669551:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:1.0:1787829836.669552:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1787829836.669556:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00080000:1.0:1787829836.669560:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829836.669561:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829836.669565:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000001:0.0:1787829836.669566:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829836.669568:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:0.0:1787829836.669571:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787829836.669573:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829836.669574:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:0.0:1787829836.669592:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829836.669597:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00100000:1.0:1787829836.669600:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327383936:192.168.203.116@tcp:250:kworker.0
00000100:00000040:0.0:1787829836.669604:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669608:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829836.669608:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.669611:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829836.669613:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829836.669617:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327383936/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829836.669618:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829836.669624:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829836.669629:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669632:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669632:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829836.669635:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:0.0:1787829836.669636:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:1.0:1787829836.669638:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829836.669641:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829836.669642:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:0.0:1787829836.669646:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829836.669646:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829836.669649:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000100:00000001:0.0:1787829836.669651:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000020:00000001:1.0:1787829836.669651:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
00000008:00100000:0.0:1787829836.669678:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829836.669681:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829836.669683:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829836.669685:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
00000100:00000010:0.0:1787829836.669687:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716de00.
02000000:00000001:1.0:1787829836.669689:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000040:0.0:1787829836.669692:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829836.669692:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829836.669694:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
02000000:00000001:0.0:1787829836.669697:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829836.669698:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829836.669701:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829836.669702:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669703:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829836.669706:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829836.669708:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787829836.669710:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1787829836.669711:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669712:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669716:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829836.669717:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829836.669718:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000010:0.0:1787829836.669719:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533000.
00000100:00000040:1.0:1787829836.669721:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829836.669725:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669729:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829836.669731:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.669733:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:0.0:1787829836.669735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787829836.669735:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829836.669739:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829836.669744:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384064:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829836.669749:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829836.669751:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787829836.669754:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787829836.669754:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.669757:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787829836.669758:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829836.669760:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829836.669763:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829836.669766:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c17e5d1d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829836.669767:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829836.669771:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829836.669775:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829836.669779:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384064, portal 10
00000100:00000001:1.0:1787829836.669783:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829836.669784:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000001:0.0:1787829836.669785:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829836.669791:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829836.669792:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3994)
00000100:00000001:1.0:1787829836.669798:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829836.669800:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829836.669802:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829836.669804:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327384064, offset 0
00000400:00000200:1.0:1787829836.669807:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829836.669810:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.669813:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829836.669815:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829836.669817:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829836.669818:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829836.669820:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829836.669822:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40772:40772:256:4294967295] 192.168.203.116@tcp LPNI seq info [40772:40772:8:4294967295]
00000400:00000200:1.0:1787829836.669824:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.669828:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829836.669831:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.669832:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829836.669835:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829836.669837:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829836.669839:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829836.669841:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787829836.669843:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.669845:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829836.669846:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829836.669848:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829836.669850:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829836.669851:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829836.669853:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829836.669855:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669859:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829836.669861:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669865:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669866:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829836.669868:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829836.669870:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829836.669872:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829836.669874:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298656779
00000100:00000001:1.0:1787829836.669876:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298657452
00000100:00000001:1.0:1787829836.669877:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298656779
00000100:00000001:1.0:1787829836.669878:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298657452
00000100:00000001:1.0:1787829836.669880:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829836.669881:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.669882:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829836.669884:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836]  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829836.669890:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829836.669892:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829836.669893:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829836.669895:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829836.669897:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829836.669899:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829836.669902:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829836.669909:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669916:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829836.669917:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.669919:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.669920:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829836.669924:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669930:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829836.669934:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669938:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829836.669939:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.669941:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.669942:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.669943:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829836.669944:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829836.669946:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829836.669948:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829836.669949:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829836.669951:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829836.669955:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384064:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829836.669958:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829836.669960:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.669965:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829836.669967:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.669968:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829836.669970:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829836.669972:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829836.669975:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384192:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829836.669978:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829836.669979:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829836.669981:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.669983:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829836.669986:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829836.669987:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829836.669991:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829836.669993:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829836.669997:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829836.669999:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384192, portal 4
00000100:00000001:1.0:1787829836.670001:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829836.670003:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829836.670008:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670012:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829836.670013:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829836.670015:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829836.670017:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327384192, offset 0
00000400:00000200:1.0:1787829836.670019:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829836.670022:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.670024:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829836.670025:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829836.670027:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829836.670028:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829836.670030:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829836.670031:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40773:40773:256:4294967295] 192.168.203.116@tcp LPNI seq info [40773:40773:8:4294967295]
00000400:00000200:1.0:1787829836.670034:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.670038:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829836.670040:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.670042:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829836.670044:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829836.670045:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829836.670048:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829836.670049:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787829836.670052:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.670053:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829836.670055:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829836.670057:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829836.670058:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829836.670059:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829836.670061:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829836.670063:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670067:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829836.670069:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670073:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.670074:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829836.670076:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829836.670077:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829836.670079:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829836.670080:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298656786
00000100:00000001:1.0:1787829836.670082:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298657452
00000100:00000001:1.0:1787829836.670083:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298656786
00000100:00000001:1.0:1787829836.670085:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298657452
00000100:00000001:1.0:1787829836.670086:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829836.670088:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.670089:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829836.670093:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836]  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829836.670100:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829836.670101:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829836.670103:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829836.670105:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829836.670106:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829836.670108:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829836.670111:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829836.670118:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670124:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829836.670126:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670127:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.670128:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829836.670132:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670138:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829836.670142:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670147:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829836.670150:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670151:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.670153:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670154:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829836.670155:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829836.670158:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829836.670161:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829836.670163:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829836.670166:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829836.670173:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384192:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829836.670175:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670177:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829836.670179:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327384064/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670185:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829836.670186:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829836.670188:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7800.
02000000:00000001:1.0:1787829836.670190:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829836.670191:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829836.670192:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829836.670194:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829836.670196:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531000.
02000000:00000001:1.0:1787829836.670198:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829836.670199:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829836.670200:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829836.670203:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829836.670204:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670205:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829836.670208:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327384192/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670214:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829836.670215:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829836.670216:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829836.670218:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829836.670219:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829836.670221:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829836.670223:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829836.670224:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532400.
02000000:00000001:1.0:1787829836.670226:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829836.670227:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829836.670228:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829836.670230:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829836.670231:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670232:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670235:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829836.670236:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.670237:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829836.670238:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829836.670239:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829836.670241:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670245:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829836.670246:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.670248:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829836.670249:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829836.670250:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829836.670252:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384320:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829836.670254:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829836.670255:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829836.670256:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829836.670257:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829836.670259:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829836.670260:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829836.670262:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829836.670264:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829836.670266:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829836.670268:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384320, portal 4
00000100:00000001:1.0:1787829836.670269:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829836.670270:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
00000100:00000040:1.0:1787829836.670273:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670277:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829836.670278:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829836.670279:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829836.670281:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327384320, offset 0
00000400:00000200:1.0:1787829836.670283:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829836.670285:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.670287:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829836.670289:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829836.670291:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829836.670292:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829836.670293:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829836.670295:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40774:40774:256:4294967295] 192.168.203.116@tcp LPNI seq info [40774:40774:8:4294967295]
00000400:00000200:1.0:1787829836.670298:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829836.670301:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829836.670304:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.670305:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829836.670308:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829836.670310:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829836.670314:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829836.670316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787829836.670331:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829836.670333:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829836.670335:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829836.670339:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829836.670342:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829836.670343:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829836.670346:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829836.670348:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670353:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829836.670355:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670359:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.670359:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829836.670362:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829836.670363:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829836.670365:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829836.670367:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298656791
00000100:00000001:1.0:1787829836.670368:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298657452
00000100:00000001:1.0:1787829836.670370:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298656791
00000100:00000001:1.0:1787829836.670371:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298657452
00000100:00000001:1.0:1787829836.670372:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829836.670373:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.670374:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829836.670376:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836]  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829836.670380:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829836.670381:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829836.670383:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829836.670385:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829836.670386:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829836.670387:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829836.670389:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829836.670393:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670397:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829836.670397:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670398:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.670399:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829836.670401:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670404:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829836.670406:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670410:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829836.670411:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670412:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829836.670412:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670413:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829836.670414:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829836.670416:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829836.670417:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829836.670418:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829836.670420:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829836.670424:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384320:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829836.670427:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670428:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829836.670430:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327384320/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829836.670433:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829836.670434:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829836.670435:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829836.670436:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829836.670437:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829836.670438:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829836.670439:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829836.670440:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829836.670441:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829836.670442:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829836.670443:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829836.670444:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829836.670445:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670446:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829836.670448:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829836.670449:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829836.670450:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829836.670451:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829837.180755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829837.180760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829837.180762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829837.180766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829837.180768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829837.180772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829837.180773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829837.180775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829837.180777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829837.372715:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298657452, at 4298657472
00000100:00000040:2.0:1787829837.372722:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298657452, at 4298657472
00000100:00000010:3.0:1787829837.372738:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f00.
00000100:00000010:2.0:1787829837.372745:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63a20.
00010000:00000001:0.0:1787829837.500751:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829837.500756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829837.500760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829837.500761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829837.500763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829837.500764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829837.500782:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829837.500784:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829837.500786:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829838.204743:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829838.204748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829838.204750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829838.204753:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829838.204756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829838.204758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829838.204760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829838.204762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829838.204764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829838.524747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829838.524751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829838.524755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829838.524756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829838.524758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829838.524759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829838.524810:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829838.524812:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829838.524815:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829839.228760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829839.228765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829839.228767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829839.228770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829839.228773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829839.228776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829839.228777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829839.228779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829839.228781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829839.548752:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829839.548757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829839.548760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829839.548761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829839.548762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829839.548763:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829839.548842:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829839.548845:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829839.548846:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829840.252815:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829840.252820:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829840.252826:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829840.252832:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829840.252834:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829840.252837:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829840.252839:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829840.252841:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829840.252843:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829840.572787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829840.572791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829840.572795:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829840.572797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829840.572800:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829840.572803:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829840.572805:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829840.572807:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829840.572810:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829840.572811:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829840.572812:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829840.572814:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829840.572838:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829840.572840:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829840.572843:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829841.276769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829841.276776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829841.276778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829841.276783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829841.276786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829841.276790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829841.276793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829841.276796:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829841.276798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829841.596728:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829841.596734:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829841.596738:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829841.596739:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829841.596742:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829841.596743:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829841.596784:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829841.596787:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829841.596789:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829841.788741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829841.788748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829841.788750:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829841.788752:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829841.788756:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829841.788761:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3989
00000400:00000200:1.0:1787829841.788767:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829841.788772:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829841.788774:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829841.788776:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788778:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.788781:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788782:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829841.788786:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829841.788787:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829841.788789:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829841.788791:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829841.788792:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829841.788794:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829841.788795:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788796:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.788797:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788798:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829841.788801:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829841.788803:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788805:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829841.788807:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829841.788809:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829841.788811:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829841.788813:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829841.788814:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829841.788821:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829841.788823:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829841.788826:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829841.788827:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829841.788829:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829841.788831:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829841.788834:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829841.788840:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829841.788845:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829841.788852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829841.788858:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829841.788861:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829841.788864:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829841.788882:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829841.788884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829841.788886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829841.788887:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829841.788888:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829841.788890:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829841.788891:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3989
00000400:00000200:1.0:1787829841.788893:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829841.788895:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829841.788896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829841.788898:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788900:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.788902:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788903:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829841.788904:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829841.788905:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829841.788906:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829841.788907:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829841.788909:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829841.788910:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787829841.788912:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829841.788914:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787829841.788916:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829841.788918:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829841.788919:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.788922:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788925:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829841.788926:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829841.788927:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829841.788932:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829841.788941:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.788943:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829841.788945:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829841.788945:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829841.788950:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829841.788951:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829841.788952:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829841.788953:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829841.788958:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829841.788959:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327384448:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829841.788961:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829841.788963:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829841.788967:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:1.0:1787829841.788967:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829841.788970:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.788973:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:1.0:1787829841.788973:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829841.788977:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829841.788978:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829841.788981:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829841.788982:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000400:00000200:0.0:1787829841.788986:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:1.0:1787829841.788987:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:0.0:1787829841.788990:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
02000000:00000001:1.0:1787829841.788991:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829841.788994:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829841.788996:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829841.788997:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829841.789000:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829841.789002:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829841.789003:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384448, portal 25
00000100:00000001:0.0:1787829841.789007:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829841.789008:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
02000000:00000010:1.0:1787829841.789010:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000040:0.0:1787829841.789012:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829841.789015:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789021:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829841.789025:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00100000:1.0:1787829841.789025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829841.789027:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829841.789032:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327384448, offset 0
00000100:00000001:1.0:1787829841.789038:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829841.789043:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789043:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829841.789050:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829841.789055:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829841.789063:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829841.789066:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:1.0:1787829841.789068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829841.789073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829841.789074:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829841.789075:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829841.789079:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829841.789081:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:1.0:1787829841.789083:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829841.789087:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1787829841.789088:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3989
00000400:00000200:0.0:1787829841.789092:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789096:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40775:40775:256:4294967295] 192.168.203.116@tcp LPNI seq info [40775:40775:8:4294967295]
00000400:00000200:0.0:1787829841.789099:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789110:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000400:00000200:1.0:1787829841.789110:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829841.789117:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829841.789120:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:0.0:1787829841.789122:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829841.789124:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789127:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.789128:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789130:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829841.789130:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829841.789132:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829841.789135:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829841.789136:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829841.789137:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829841.789140:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829841.789142:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829841.789144:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789146:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.789147:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789148:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.0:1787829841.789149:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:1.0:1787829841.789150:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829841.789154:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.0:1787829841.789156:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1787829841.789157:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829841.789161:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829841.789165:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.1:1787829841.789165:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:1.0:1787829841.789168:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829841.789169:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1787829841.789170:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829841.789174:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000800:00000100:0.0:1787829841.789176:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000008:00000001:1.0:1787829841.789177:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829841.789179:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829841.789180:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829841.789185:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000400:00000200:0.0:1787829841.789186:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1787829841.789187:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829841.789191:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829841.789192:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1787829841.789193:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829841.789196:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00000001:1.0:1787829841.789196:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829841.789197:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1787829841.789199:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
02000000:00000001:1.0:1787829841.789200:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789203:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:1.0:1787829841.789205:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000200:0.0:1787829841.789209:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829841.789210:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789221:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:1.0:1787829841.789222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829841.789227:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829841.789230:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829841.789234:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829841.789239:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789242:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789247:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829841.789248:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829841.789251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829841.789253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829841.789254:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829841.789254:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829841.789256:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829841.789259:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829841.789262:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829841.789263:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3989
00000100:00000001:0.0:1787829841.789267:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298662572
00000400:00000200:1.0:1787829841.789272:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:0.0:1787829841.789275:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc6c0.
00000400:00000200:1.0:1787829841.789278:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829841.789280:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829841.789282:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829841.789284:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829841.789285:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.789288:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789290:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829841.789291:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829841.789292:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829841.789293:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298662572
00000020:00000001:1.0:1787829841.789294:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829841.789296:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829841.789298:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000010:0.0:1787829841.789300:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccde0.
00000020:00000001:1.0:1787829841.789300:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829841.789305:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000040:1.0:1787829841.789306:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789310:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829841.789311:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829841.789313:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829841.789314:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829841.789316:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829841.789317:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829841.789317:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829841.789321:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829841.789322:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829841.789325:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829841.789326:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1787829841.789330:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829841.789334:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841]  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829841.789334:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829841.789336:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829841.789342:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829841.789346:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829841.789353:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000010:1.0:1787829841.789353:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000400:00000001:0.0:1787829841.789357:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:1.0:1787829841.789358:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000010:0.0:1787829841.789360:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
02000000:00000001:1.0:1787829841.789362:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829841.789364:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829841.789366:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829841.789368:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829841.789369:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829841.789371:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829841.789374:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
02000000:00000010:1.0:1787829841.789374:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:0.0:1787829841.789376:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829841.789379:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829841.789381:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829841.789386:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829841.789398:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829841.789398:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829841.789402:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829841.789409:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789410:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829841.789412:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789416:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.789418:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829841.789420:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829841.789424:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829841.789425:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3999)
00000100:00000001:0.0:1787829841.789432:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829841.789436:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789440:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829841.789443:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789444:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.789445:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789446:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829841.789447:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829841.789449:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829841.789451:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829841.789452:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829841.789455:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829841.789459:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327384448:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829841.789462:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789463:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.789465:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327384448/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789469:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829841.789471:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829841.789473:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829841.789474:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829841.789475:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829841.789476:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829841.789478:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829841.789479:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829841.789482:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829841.789483:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829841.789484:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829841.789486:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829841.789487:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789488:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789493:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829841.789494:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789495:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829841.789497:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829841.789498:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829841.789500:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789504:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829841.789504:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.789506:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829841.789507:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829841.789508:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829841.789510:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384576:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829841.789512:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829841.789513:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829841.789514:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.789515:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829841.789517:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829841.789518:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829841.789520:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829841.789522:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829841.789524:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829841.789526:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384576, portal 10
00000100:00000001:0.0:1787829841.789527:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829841.789528:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829841.789532:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789536:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829841.789537:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829841.789538:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829841.789554:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327384576, offset 0
00000400:00000200:0.0:1787829841.789558:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789561:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789564:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829841.789566:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829841.789567:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829841.789569:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829841.789570:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789572:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40776:40776:256:4294967295] 192.168.203.116@tcp LPNI seq info [40776:40776:8:4294967295]
00000400:00000200:0.0:1787829841.789575:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789579:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829841.789581:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789583:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829841.789587:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829841.789589:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829841.789594:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829841.789596:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829841.789599:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789601:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829841.789603:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829841.789607:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829841.789609:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829841.789611:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829841.789613:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829841.789617:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789625:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.789628:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789632:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789633:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829841.789636:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829841.789638:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829841.789641:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829841.789642:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298661898
00000100:00000001:0.0:1787829841.789644:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298662572
00000100:00000001:0.0:1787829841.789645:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298661898
00000100:00000001:0.0:1787829841.789647:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298662572
00000100:00000001:0.0:1787829841.789649:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829841.789650:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789651:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829841.789654:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841]  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829841.789673:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829841.789674:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829841.789675:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829841.789677:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829841.789678:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829841.789679:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829841.789681:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829841.789686:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789690:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829841.789690:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789691:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.789692:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829841.789694:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789697:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829841.789699:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789703:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829841.789705:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789706:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.789707:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789708:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829841.789709:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829841.789711:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829841.789713:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829841.789714:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829841.789716:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829841.789721:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384576:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829841.789723:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829841.789724:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789728:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829841.789729:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.789730:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829841.789731:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829841.789733:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829841.789734:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384704:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829841.789736:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829841.789737:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829841.789738:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.789739:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829841.789741:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829841.789742:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829841.789744:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829841.789746:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829841.789748:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829841.789749:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384704, portal 4
00000100:00000001:0.0:1787829841.789751:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829841.789751:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829841.789755:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789759:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829841.789760:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829841.789761:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829841.789763:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327384704, offset 0
00000400:00000200:0.0:1787829841.789765:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789767:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789769:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829841.789770:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829841.789772:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829841.789773:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829841.789775:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789776:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40777:40777:256:4294967295] 192.168.203.116@tcp LPNI seq info [40777:40777:8:4294967295]
00000400:00000200:0.0:1787829841.789779:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789783:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829841.789786:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789788:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829841.789790:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829841.789798:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829841.789801:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829841.789803:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829841.789806:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789807:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829841.789809:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829841.789810:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829841.789812:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829841.789813:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829841.789815:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829841.789817:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789821:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.789823:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789833:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789834:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829841.789836:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829841.789838:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829841.789840:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829841.789841:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298661906
00000100:00000001:0.0:1787829841.789842:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298662572
00000100:00000001:0.0:1787829841.789844:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298661906
00000100:00000001:0.0:1787829841.789845:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298662572
00000100:00000001:0.0:1787829841.789846:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829841.789847:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789848:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829841.789850:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841]  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829841.789854:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829841.789855:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829841.789856:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829841.789858:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829841.789859:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829841.789860:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829841.789861:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829841.789866:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789869:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829841.789870:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789871:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.789872:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829841.789874:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789878:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829841.789880:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789884:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829841.789885:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789886:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.789886:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.789891:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829841.789892:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829841.789893:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829841.789895:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829841.789898:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829841.789900:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829841.789905:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384704:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829841.789907:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829841.789909:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789913:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829841.789913:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.789914:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829841.789915:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829841.789916:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829841.789918:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384832:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829841.789919:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829841.789920:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829841.789922:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829841.789923:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829841.789925:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829841.789926:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829841.789927:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829841.789929:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829841.789930:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829841.789932:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384832, portal 4
00000100:00000001:0.0:1787829841.789933:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829841.789934:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:0.0:1787829841.789937:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789940:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829841.789942:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829841.789943:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829841.789944:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327384832, offset 0
00000400:00000200:0.0:1787829841.789946:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789948:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789950:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829841.789951:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829841.789952:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829841.789954:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829841.789955:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829841.789957:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40778:40778:256:4294967295] 192.168.203.116@tcp LPNI seq info [40778:40778:8:4294967295]
00000400:00000200:0.0:1787829841.789959:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829841.789963:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829841.789965:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789967:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829841.789968:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829841.789970:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787829841.789972:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829841.789974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787829841.789976:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829841.789977:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829841.789979:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829841.789981:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829841.789982:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829841.789983:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829841.789984:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829841.789986:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789990:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.789992:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.789996:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.789997:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829841.789998:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829841.790000:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829841.790001:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829841.790002:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298661911
00000100:00000001:0.0:1787829841.790004:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298662572
00000100:00000001:0.0:1787829841.790005:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298661911
00000100:00000001:0.0:1787829841.790006:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298662572
00000100:00000001:0.0:1787829841.790007:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829841.790008:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.790009:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829841.790011:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841]  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829841.790015:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829841.790016:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829841.790016:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829841.790017:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829841.790018:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829841.790020:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829841.790021:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829841.790026:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.790030:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829841.790030:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790031:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.790032:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829841.790034:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.790037:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829841.790040:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.790047:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829841.790049:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790051:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829841.790052:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790053:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829841.790054:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829841.790057:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829841.790060:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829841.790062:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829841.790066:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829841.790074:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327384832:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829841.790077:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790079:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.790082:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327384576/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.790088:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829841.790089:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829841.790090:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829841.790092:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829841.790092:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829841.790093:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829841.790095:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829841.790096:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829841.790097:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829841.790098:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829841.790099:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829841.790101:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829841.790102:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790103:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.790107:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327384704/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.790113:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829841.790114:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829841.790115:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829841.790117:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829841.790118:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829841.790119:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829841.790121:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829841.790122:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829841.790124:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829841.790125:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829841.790126:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829841.790128:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829841.790129:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790131:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829841.790134:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327384832/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829841.790141:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829841.790143:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829841.790144:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829841.790147:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829841.790147:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829841.790148:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829841.790149:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829841.790150:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829841.790151:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829841.790152:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829841.790153:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829841.790154:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829841.790154:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790155:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829841.790158:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829841.790158:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829841.790159:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829841.790160:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829842.300769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829842.300776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829842.300778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829842.300782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829842.300785:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829842.300788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829842.300789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829842.300791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829842.300793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829842.492706:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298662572, at 4298662592
00000100:00000040:2.0:1787829842.492709:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298662572, at 4298662592
00000100:00000010:2.0:1787829842.492720:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000100:00000010:3.0:1787829842.492726:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00010000:00000001:0.0:1787829842.620744:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829842.620749:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829842.620753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829842.620754:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829842.620757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829842.620758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829842.620801:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829842.620805:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829842.620809:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829843.324752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829843.324757:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829843.324758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829843.324762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829843.324764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829843.324766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829843.324768:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829843.324770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829843.324772:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829843.644728:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829843.644732:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829843.644736:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829843.644737:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829843.644740:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829843.644741:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829843.644755:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829843.644756:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829843.644758:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829844.348765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829844.348773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829844.348774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829844.348779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829844.348783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829844.348786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829844.348789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829844.348791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829844.348794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829844.668794:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829844.668801:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829844.668809:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829844.668811:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829844.668816:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829844.668818:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829844.668878:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829844.668882:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829844.668886:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829845.372722:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829845.372729:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829845.372731:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829845.372736:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829845.372739:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829845.372742:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829845.372744:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829845.372746:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829845.372749:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692750:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829845.692756:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829845.692758:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829845.692760:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692763:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829845.692765:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692767:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692768:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829845.692772:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829845.692773:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829845.692774:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692775:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829845.692775:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692776:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692777:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829845.692779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829845.692780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829845.692832:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829845.692835:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829845.692837:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829846.396755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829846.396762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829846.396764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829846.396769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829846.396771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829846.396775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829846.396777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829846.396780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829846.396782:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829846.716735:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829846.716740:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829846.716744:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829846.716745:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829846.716747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829846.716748:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829846.716762:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829846.716763:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829846.716766:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829846.908740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829846.908747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829846.908749:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829846.908752:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829846.908756:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829846.908760:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3994
00000400:00000200:1.0:1787829846.908768:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829846.908773:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829846.908775:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829846.908777:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908779:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.908782:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908783:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829846.908786:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829846.908787:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829846.908789:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829846.908792:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829846.908794:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829846.908795:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829846.908804:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908805:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.908806:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908807:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829846.908810:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829846.908812:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908813:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829846.908815:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829846.908818:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829846.908821:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.908822:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829846.908825:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829846.908833:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829846.908837:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829846.908840:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829846.908842:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.908851:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829846.908853:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829846.908857:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829846.908865:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829846.908869:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829846.908876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829846.908883:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829846.908886:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829846.908889:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829846.908906:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829846.908908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829846.908910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829846.908911:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829846.908912:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829846.908914:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829846.908915:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3994
00000400:00000200:1.0:1787829846.908917:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829846.908919:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829846.908920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829846.908922:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908924:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.908925:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908926:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829846.908927:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829846.908928:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829846.908930:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829846.908932:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829846.908933:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829846.908935:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829846.908938:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908939:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.908940:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908942:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829846.908943:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829846.908944:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829846.908946:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829846.908947:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829846.908951:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829846.908952:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.908953:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829846.908955:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00100000:1.0:1787829846.908956:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829846.908959:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.908960:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829846.908961:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.908962:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:1.0:1787829846.908965:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829846.908968:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000040:0.0:1787829846.908970:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829846.908971:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829846.908973:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.908978:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829846.908981:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829846.908984:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829846.908985:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.908986:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.908991:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829846.908993:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:1.0:1787829846.908994:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829846.908996:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829846.909001:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327384960:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829846.909001:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909005:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829846.909011:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:1.0:1787829846.909012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829846.909014:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.909020:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829846.909026:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000010:0.0:1787829846.909027:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
00000100:00000001:1.0:1787829846.909028:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.909033:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829846.909035:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829846.909042:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829846.909048:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000001:1.0:1787829846.909051:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829846.909056:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00080000:1.0:1787829846.909056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829846.909062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:0.0:1787829846.909063:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327384960, portal 25
00000100:00000001:1.0:1787829846.909064:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829846.909067:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1787829846.909068:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829846.909071:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000001:1.0:1787829846.909072:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:0.0:1787829846.909080:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829846.909090:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3994
00000100:00000001:0.0:1787829846.909092:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787829846.909097:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829846.909098:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829846.909101:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000010:0.0:1787829846.909104:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00080000:1.0:1787829846.909104:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829846.909109:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829846.909110:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327384960, offset 0
00000100:00000001:1.0:1787829846.909111:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.909115:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909117:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829846.909118:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829846.909119:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829846.909122:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829846.909123:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:1.0:1787829846.909124:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829846.909126:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829846.909127:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829846.909130:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909131:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.909132:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909133:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829846.909135:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829846.909138:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909139:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829846.909141:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829846.909143:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829846.909144:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829846.909147:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.909150:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829846.909152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829846.909154:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000008:00000001:1.0:1787829846.909160:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909162:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829846.909163:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909168:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000010:1.0:1787829846.909170:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000400:00000200:0.0:1787829846.909174:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000040:1.0:1787829846.909176:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829846.909179:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:1.0:1787829846.909181:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829846.909185:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909186:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40779:40779:256:4294967295] 192.168.203.116@tcp LPNI seq info [40779:40779:8:4294967295]
00000100:00000001:1.0:1787829846.909189:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829846.909192:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829846.909195:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909196:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:1.0:1787829846.909202:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829846.909208:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909212:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829846.909221:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00100000:1.0:1787829846.909222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1787829846.909227:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829846.909229:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829846.909231:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829846.909235:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829846.909239:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829846.909246:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1787829846.909246:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829846.909249:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829846.909253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829846.909257:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829846.909258:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1787829846.909258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829846.909264:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1787829846.909265:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00080000:1.0:1787829846.909267:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3994
00000400:00000200:1.0:1787829846.909273:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000100:0.0:1787829846.909277:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829846.909279:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829846.909283:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000010:0.0:1787829846.909284:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00000040:1.0:1787829846.909286:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909289:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829846.909291:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909292:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829846.909293:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1787829846.909294:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829846.909296:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829846.909298:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829846.909300:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829846.909303:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829846.909304:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829846.909310:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000020:00000001:1.0:1787829846.909310:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000010:0.0:1787829846.909314:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000040:1.0:1787829846.909316:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909319:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829846.909321:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787829846.909322:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909326:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829846.909329:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829846.909330:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829846.909338:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829846.909344:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909347:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1787829846.909348:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829846.909352:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787829846.909354:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829846.909356:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.909361:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829846.909365:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829846.909370:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909371:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.909371:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909377:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:1.0:1787829846.909378:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829846.909381:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829846.909383:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829846.909384:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829846.909384:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829846.909389:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829846.909390:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829846.909392:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909393:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:1.0:1787829846.909396:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:0.0:1787829846.909398:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298667692
00000100:00000001:1.0:1787829846.909400:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829846.909406:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc660.
00000100:00100000:1.0:1787829846.909407:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829846.909414:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829846.909416:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909417:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787829846.909418:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909424:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298667692
00000100:00000001:1.0:1787829846.909425:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829846.909430:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf60.
00000100:00000040:1.0:1787829846.909431:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4004)
00000100:00000001:0.0:1787829846.909437:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829846.909442:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829846.909445:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909448:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829846.909453:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846]  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829846.909462:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829846.909464:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829846.909466:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829846.909469:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829846.909471:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829846.909473:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829846.909476:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829846.909483:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909490:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829846.909492:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909494:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.909496:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829846.909500:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909520:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829846.909525:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909532:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829846.909535:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909536:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.909537:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909539:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829846.909541:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829846.909544:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829846.909547:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829846.909549:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829846.909553:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829846.909560:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327384960:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829846.909565:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909567:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.909570:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327384960/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909577:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829846.909579:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829846.909581:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829846.909584:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829846.909586:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829846.909588:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829846.909590:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829846.909592:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:0.0:1787829846.909595:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829846.909597:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829846.909599:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829846.909602:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829846.909603:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909605:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909611:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829846.909612:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909614:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829846.909616:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829846.909617:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829846.909620:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909625:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829846.909626:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.909627:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829846.909628:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829846.909630:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829846.909631:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385088:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829846.909633:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829846.909635:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829846.909636:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.909637:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829846.909640:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829846.909641:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909651:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829846.909653:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829846.909673:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829846.909676:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385088, portal 10
00000100:00000001:0.0:1787829846.909677:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829846.909678:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829846.909683:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909687:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829846.909688:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829846.909689:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829846.909691:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327385088, offset 0
00000400:00000200:0.0:1787829846.909693:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829846.909696:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829846.909698:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829846.909700:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829846.909702:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829846.909703:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829846.909705:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829846.909707:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40780:40780:256:4294967295] 192.168.203.116@tcp LPNI seq info [40780:40780:8:4294967295]
00000400:00000200:0.0:1787829846.909709:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829846.909713:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829846.909716:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829846.909718:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829846.909720:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829846.909722:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829846.909725:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829846.909727:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1787829846.909729:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829846.909730:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829846.909733:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829846.909735:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829846.909737:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829846.909738:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829846.909739:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829846.909741:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909746:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.909748:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909752:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909752:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829846.909754:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829846.909757:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829846.909759:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829846.909760:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298667019
00000100:00000001:0.0:1787829846.909762:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298667692
00000100:00000001:0.0:1787829846.909763:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298667019
00000100:00000001:0.0:1787829846.909764:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298667692
00000100:00000001:0.0:1787829846.909765:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829846.909766:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909774:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829846.909776:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846]  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829846.909781:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829846.909781:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829846.909783:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829846.909784:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829846.909785:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829846.909786:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829846.909788:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829846.909793:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909800:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829846.909801:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909802:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.909803:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829846.909805:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909809:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829846.909811:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909815:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829846.909816:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909817:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.909818:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909819:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829846.909820:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829846.909822:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829846.909824:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829846.909825:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829846.909827:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829846.909831:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385088:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829846.909833:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829846.909835:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909843:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829846.909843:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.909845:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829846.909846:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829846.909847:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829846.909849:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385216:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829846.909850:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829846.909851:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829846.909852:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.909853:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829846.909855:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829846.909857:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.909858:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829846.909860:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829846.909861:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829846.909863:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385216, portal 4
00000100:00000001:0.0:1787829846.909864:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829846.909865:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829846.909868:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909872:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829846.909873:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829846.909874:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829846.909876:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327385216, offset 0
00000400:00000200:0.0:1787829846.909877:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829846.909879:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829846.909881:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829846.909883:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829846.909884:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829846.909885:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829846.909887:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829846.909889:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40781:40781:256:4294967295] 192.168.203.116@tcp LPNI seq info [40781:40781:8:4294967295]
00000400:00000200:0.0:1787829846.909891:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829846.909895:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829846.909897:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829846.909899:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829846.909901:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829846.909904:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829846.909908:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829846.909910:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1787829846.909914:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829846.909916:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829846.909918:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829846.909921:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829846.909923:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829846.909925:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829846.909927:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829846.909931:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909938:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.909941:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909947:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909949:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829846.909951:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829846.909954:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829846.909956:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829846.909958:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298667026
00000100:00000001:0.0:1787829846.909960:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298667692
00000100:00000001:0.0:1787829846.909961:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298667026
00000100:00000001:0.0:1787829846.909962:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298667692
00000100:00000001:0.0:1787829846.909964:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829846.909965:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.909966:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829846.909969:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846]  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829846.909974:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829846.909975:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829846.909976:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829846.909977:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829846.909978:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829846.909979:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829846.909981:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829846.909986:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909990:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829846.909990:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.909991:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.909992:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829846.909994:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.909997:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829846.909999:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910003:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829846.910006:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910007:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.910008:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910009:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829846.910011:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829846.910013:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829846.910016:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829846.910018:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829846.910021:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829846.910029:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385216:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829846.910033:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829846.910036:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910042:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829846.910043:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.910045:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829846.910047:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829846.910049:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829846.910050:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385344:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829846.910052:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829846.910054:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829846.910054:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829846.910056:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829846.910058:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829846.910059:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829846.910061:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829846.910063:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829846.910065:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829846.910067:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385344, portal 4
00000100:00000001:0.0:1787829846.910068:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829846.910069:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:0.0:1787829846.910072:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910076:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829846.910078:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829846.910079:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829846.910080:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327385344, offset 0
00000400:00000200:0.0:1787829846.910083:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829846.910085:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829846.910087:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829846.910089:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829846.910090:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829846.910092:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829846.910093:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829846.910095:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40782:40782:256:4294967295] 192.168.203.116@tcp LPNI seq info [40782:40782:8:4294967295]
00000400:00000200:0.0:1787829846.910097:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829846.910101:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829846.910104:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829846.910106:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829846.910108:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829846.910109:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787829846.910112:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829846.910114:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1787829846.910116:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829846.910117:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829846.910119:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829846.910121:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829846.910123:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829846.910124:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829846.910125:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829846.910127:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910132:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.910133:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910137:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.910138:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829846.910140:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829846.910142:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829846.910145:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829846.910146:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298667031
00000100:00000001:0.0:1787829846.910148:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298667692
00000100:00000001:0.0:1787829846.910150:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298667031
00000100:00000001:0.0:1787829846.910153:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298667692
00000100:00000001:0.0:1787829846.910155:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829846.910157:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.910159:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829846.910163:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846]  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829846.910170:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829846.910171:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829846.910173:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829846.910176:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829846.910177:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829846.910179:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829846.910182:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829846.910190:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910196:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829846.910197:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910199:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.910200:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829846.910203:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910207:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829846.910209:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910214:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829846.910216:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910217:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829846.910218:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910219:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829846.910221:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829846.910222:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829846.910224:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829846.910225:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829846.910227:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829846.910231:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385344:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829846.910233:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910234:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.910236:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327385088/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910240:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829846.910241:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829846.910242:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829846.910244:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829846.910245:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829846.910246:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829846.910247:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829846.910248:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829846.910250:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829846.910251:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829846.910252:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829846.910254:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829846.910254:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910256:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.910257:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327385216/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910261:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829846.910262:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829846.910262:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532400.
02000000:00000001:0.0:1787829846.910264:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829846.910264:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829846.910265:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829846.910266:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829846.910266:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829846.910267:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829846.910268:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829846.910269:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829846.910270:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829846.910271:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910273:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829846.910275:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327385344/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829846.910280:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829846.910281:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829846.910282:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829846.910284:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829846.910285:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829846.910286:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829846.910288:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829846.910289:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829846.910291:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829846.910292:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829846.910294:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829846.910295:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829846.910297:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910298:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829846.910301:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829846.910303:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829846.910305:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829846.910307:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829847.420804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829847.420814:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829847.420817:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829847.420824:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829847.420828:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829847.420832:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829847.420835:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829847.420841:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829847.420845:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829847.612758:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298667692, at 4298667712
00000100:00000040:2.0:1787829847.612760:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298667692, at 4298667712
00000100:00000010:3.0:1787829847.612774:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00000100:00000010:2.0:1787829847.612782:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00010000:00000001:0.0:1787829847.740740:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829847.740748:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829847.740753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829847.740755:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829847.740758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829847.740760:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829847.740801:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829847.740804:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829847.740807:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829848.444738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829848.444744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829848.444746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829848.444752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829848.444756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829848.444760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829848.444762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829848.444765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829848.444768:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829848.764770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829848.764776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829848.764784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829848.764786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829848.764789:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829848.764791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829848.764848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829848.764851:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829848.764854:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829849.468788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829849.468796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829849.468798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829849.468803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829849.468807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829849.468811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829849.468813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829849.468816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829849.468818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829849.788772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829849.788779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829849.788785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829849.788786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829849.788789:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829849.788790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829849.788811:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829849.788813:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829849.788816:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829850.492739:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829850.492745:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829850.492747:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829850.492752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829850.492755:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829850.492758:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829850.492760:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829850.492762:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829850.492764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829850.812798:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829850.812803:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829850.812808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829850.812809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829850.812812:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829850.812816:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829850.812818:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829850.812820:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829850.812822:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829850.812824:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829850.812826:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829850.812827:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829850.812881:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829850.812884:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829850.812887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829851.516779:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829851.516785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829851.516788:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829851.516793:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829851.516797:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829851.516800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829851.516803:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829851.516806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829851.516809:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829851.836771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829851.836777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829851.836785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829851.836787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829851.836792:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829851.836793:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829851.836846:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829851.836849:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829851.836852:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829852.028784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829852.028795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829852.028798:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829852.028801:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829852.028805:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829852.028812:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 3999
00000400:00000200:2.0:1787829852.028821:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829852.028827:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829852.028831:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829852.028834:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.028837:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829852.028839:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.028842:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829852.028845:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829852.028847:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829852.028850:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829852.028854:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829852.028856:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829852.028858:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829852.028861:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.028863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829852.028865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.028867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829852.028870:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829852.028873:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829852.028876:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829852.028879:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829852.028882:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829852.028885:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.028888:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829852.028890:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829852.028900:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829852.028904:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829852.028909:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829852.028910:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.028913:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829852.028915:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829852.028920:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829852.028928:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829852.028935:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829852.028945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829852.028956:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829852.028961:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829852.028966:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829852.028986:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829852.028988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829852.028991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829852.028992:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829852.028993:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829852.028996:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829852.028997:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3999
00000400:00000200:2.0:1787829852.029000:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829852.029003:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829852.029005:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829852.029007:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029008:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829852.029010:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029011:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829852.029013:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829852.029014:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829852.029015:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829852.029017:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829852.029019:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829852.029022:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829852.029026:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029029:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829852.029031:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029034:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829852.029036:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829852.029039:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1787829852.029040:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829852.029043:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829852.029046:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029047:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1787829852.029051:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829852.029056:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829852.029059:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829852.029066:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029071:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829852.029071:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829852.029075:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787829852.029076:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829852.029078:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1787829852.029081:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029082:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829852.029085:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029086:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829852.029090:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000100:00100000:0.0:1787829852.029091:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327385472:192.168.203.116@tcp:250:kworker.0
00000020:00000040:2.0:1787829852.029096:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:0.0:1787829852.029097:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829852.029101:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829852.029102:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829852.029103:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029105:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029106:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829852.029108:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787829852.029109:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829852.029112:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829852.029114:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000010:2.0:1787829852.029116:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829852.029117:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029122:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829852.029126:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829852.029132:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00100000:2.0:1787829852.029132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829852.029138:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000001:2.0:1787829852.029140:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829852.029143:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829852.029146:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385472, portal 25
00000100:00000040:2.0:1787829852.029147:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029150:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829852.029152:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00000040:0.0:1787829852.029162:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829852.029172:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829852.029177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829852.029182:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787829852.029183:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829852.029186:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1787829852.029187:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829852.029188:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:0.0:1787829852.029190:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:2.0:1787829852.029191:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:0.0:1787829852.029195:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327385472, offset 0
00000100:00080000:2.0:1787829852.029196:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3999
00000400:00000200:2.0:1787829852.029202:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.029206:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829852.029209:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829852.029212:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829852.029214:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829852.029217:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829852.029217:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829852.029221:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029223:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829852.029227:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000001:2.0:1787829852.029227:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829852.029230:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829852.029233:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829852.029235:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829852.029236:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1787829852.029239:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829852.029243:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:2.0:1787829852.029243:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829852.029248:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1787829852.029249:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829852.029253:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029253:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829852.029257:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829852.029262:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40783:40783:256:4294967295] 192.168.203.116@tcp LPNI seq info [40783:40783:8:4294967295]
00000100:00000001:2.0:1787829852.029262:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829852.029267:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.029276:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787829852.029276:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829852.029284:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.0:1787829852.029285:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029288:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829852.029291:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
02000000:00000001:2.0:1787829852.029292:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829852.029295:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.0:1787829852.029300:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829852.029300:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029302:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1787829852.029306:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000008:00100000:2.0:1787829852.029306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829852.029312:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029316:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829852.029316:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000010:2.0:1787829852.029320:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000800:00000200:0.1:1787829852.029321:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000020:00000040:2.0:1787829852.029326:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000100:0.0:1787829852.029329:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829852.029331:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:0.0:1787829852.029332:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000001:2.0:1787829852.029334:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029336:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1787829852.029338:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787829852.029338:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829852.029340:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829852.029344:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000010:2.0:1787829852.029345:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829852.029349:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829852.029350:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829852.029353:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00100000:2.0:1787829852.029356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829852.029358:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829852.029364:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829852.029367:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829852.029371:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829852.029377:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029381:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829852.029385:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829852.029389:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787829852.029390:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829852.029393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829852.029395:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829852.029397:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829852.029401:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829852.029406:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 3999
00000100:00000001:0.0:1787829852.029410:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829852.029411:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829852.029416:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:2.0:1787829852.029417:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829852.029421:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829852.029423:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029425:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829852.029426:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829852.029427:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029431:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829852.029433:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829852.029435:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787829852.029435:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829852.029438:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829852.029441:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829852.029442:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298672812
00000020:00000001:2.0:1787829852.029443:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829852.029449:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829852.029451:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029452:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:0.0:1787829852.029453:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd20.
00000100:00000040:2.0:1787829852.029454:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029458:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829852.029460:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829852.029462:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829852.029463:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029464:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000001:2.0:1787829852.029465:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829852.029482:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829852.029485:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029487:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829852.029488:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298672812
00000008:00100000:2.0:1787829852.029489:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829852.029494:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029495:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829852.029496:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc180.
00000100:00000010:2.0:1787829852.029499:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000100:00000001:0.0:1787829852.029502:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000040:2.0:1787829852.029503:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829852.029509:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829852.029510:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.029511:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829852.029512:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829852.029513:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829852.029517:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:0.0:1787829852.029519:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829852.029521:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029524:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029529:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787829852.029531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829852.029538:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829852/real 1787829852]  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829852.029546:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829852.029551:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829852.029557:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1787829852.029561:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829852.029566:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:2.0:1787829852.029571:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829852.029572:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000040:2.0:1787829852.029575:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4009)
00000400:00000200:0.0:1787829852.029578:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829852.029582:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829852.029585:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829852.029591:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829852.029602:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029610:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829852.029612:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029614:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.029615:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829852.029618:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029623:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829852.029626:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029631:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829852.029632:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029633:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.029635:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029637:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829852.029638:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829852.029640:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829852.029643:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829852.029645:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829852.029647:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829852.029652:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327385472:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829852.029681:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029683:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.029686:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327385472/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029690:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829852.029692:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829852.029694:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829852.029696:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829852.029697:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829852.029698:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829852.029700:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829852.029702:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829852.029705:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829852.029706:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829852.029708:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:0.0:1787829852.029712:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829852.029713:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029715:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029723:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829852.029725:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029727:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829852.029729:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829852.029730:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829852.029735:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029743:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829852.029744:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029747:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829852.029748:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829852.029751:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829852.029754:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385600:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829852.029757:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829852.029759:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829852.029760:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029761:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829852.029764:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829852.029765:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829852.029768:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829852.029770:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829852.029773:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829852.029775:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385600, portal 10
00000100:00000001:0.0:1787829852.029776:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829852.029777:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:0.0:1787829852.029782:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029787:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829852.029789:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829852.029790:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829852.029792:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327385600, offset 0
00000400:00000200:0.0:1787829852.029794:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829852.029797:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.029800:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829852.029802:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829852.029804:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829852.029805:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829852.029807:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829852.029810:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40784:40784:256:4294967295] 192.168.203.116@tcp LPNI seq info [40784:40784:8:4294967295]
00000400:00000200:0.0:1787829852.029813:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.029817:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829852.029820:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829852.029822:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829852.029825:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829852.029827:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829852.029830:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829852.029832:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829852.029847:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829852.029848:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829852.029850:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829852.029853:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829852.029856:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829852.029857:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829852.029859:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829852.029861:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029867:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.029869:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029873:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029874:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829852.029876:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829852.029879:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829852.029881:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829852.029883:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298672139
00000100:00000001:0.0:1787829852.029885:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298672812
00000100:00000001:0.0:1787829852.029886:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298672139
00000100:00000001:0.0:1787829852.029888:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298672812
00000100:00000001:0.0:1787829852.029889:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829852.029891:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.029892:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829852.029895:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829852/real 1787829852]  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829852.029900:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829852.029901:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829852.029902:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829852.029903:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829852.029904:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829852.029906:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829852.029908:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829852.029913:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029917:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829852.029918:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029919:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.029920:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829852.029922:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029926:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829852.029928:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029932:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829852.029934:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029935:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.029936:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.029938:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829852.029939:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829852.029941:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829852.029943:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829852.029945:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829852.029947:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829852.029952:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385600:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829852.029955:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829852.029957:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029960:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829852.029961:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029963:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829852.029964:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829852.029965:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829852.029967:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385728:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829852.029968:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829852.029970:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829852.029971:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.029972:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829852.029974:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829852.029975:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829852.029977:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829852.029979:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829852.029981:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829852.029983:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385728, portal 4
00000100:00000001:0.0:1787829852.029984:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829852.029985:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:0.0:1787829852.029989:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.029993:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829852.029995:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829852.029997:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829852.029998:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327385728, offset 0
00000400:00000200:0.0:1787829852.030001:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829852.030003:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.030005:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829852.030007:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829852.030009:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829852.030011:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829852.030012:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829852.030014:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40785:40785:256:4294967295] 192.168.203.116@tcp LPNI seq info [40785:40785:8:4294967295]
00000400:00000200:0.0:1787829852.030017:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.030021:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829852.030024:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829852.030025:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829852.030028:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829852.030030:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829852.030032:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829852.030034:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829852.030037:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829852.030038:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829852.030040:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829852.030042:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829852.030044:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829852.030045:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829852.030046:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829852.030048:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030053:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.030055:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030059:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.030061:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829852.030063:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829852.030064:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829852.030067:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829852.030068:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298672146
00000100:00000001:0.0:1787829852.030069:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298672812
00000100:00000001:0.0:1787829852.030071:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298672146
00000100:00000001:0.0:1787829852.030072:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298672812
00000100:00000001:0.0:1787829852.030073:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829852.030075:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.030076:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829852.030078:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829852/real 1787829852]  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829852.030082:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829852.030083:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829852.030084:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829852.030086:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829852.030087:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829852.030088:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829852.030090:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829852.030095:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030099:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829852.030100:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030101:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.030101:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829852.030104:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030107:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829852.030110:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030115:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829852.030116:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030117:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.030118:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030119:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829852.030120:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829852.030122:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829852.030124:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829852.030125:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829852.030127:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829852.030132:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385728:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829852.030134:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829852.030136:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030139:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829852.030140:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.030141:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829852.030143:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829852.030144:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829852.030145:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385856:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829852.030147:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829852.030148:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829852.030149:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829852.030150:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829852.030152:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829852.030153:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829852.030155:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829852.030157:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829852.030159:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829852.030160:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385856, portal 4
00000100:00000001:0.0:1787829852.030162:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829852.030163:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:0.0:1787829852.030167:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030174:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829852.030176:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829852.030178:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829852.030181:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327385856, offset 0
00000400:00000200:0.0:1787829852.030185:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829852.030189:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.030192:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829852.030195:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829852.030198:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829852.030200:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829852.030203:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829852.030206:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40786:40786:256:4294967295] 192.168.203.116@tcp LPNI seq info [40786:40786:8:4294967295]
00000400:00000200:0.0:1787829852.030211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829852.030217:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829852.030222:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829852.030225:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829852.030227:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829852.030229:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787829852.030232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829852.030234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1787829852.030237:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829852.030238:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829852.030240:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829852.030242:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829852.030244:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829852.030245:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829852.030247:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829852.030250:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030255:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.030257:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829877 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030261:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.030262:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829852.030265:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829852.030266:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829852.030268:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829852.030270:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298672151
00000100:00000001:0.0:1787829852.030271:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298672812
00000100:00000001:0.0:1787829852.030273:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298672151
00000100:00000001:0.0:1787829852.030274:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298672812
00000100:00000001:0.0:1787829852.030276:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829852.030277:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.030279:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829852.030281:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829852/real 1787829852]  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829852.030285:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829852.030286:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829852.030287:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829852.030288:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829852.030289:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829852.030291:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829852.030292:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829852.030298:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030306:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829852.030307:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030308:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.030309:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829852.030312:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030315:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829852.030318:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030322:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829852.030323:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030324:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829852.030325:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030326:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829852.030327:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829852.030329:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829852.030331:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829852.030333:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829852.030335:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829852.030340:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327385856:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829852.030342:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030343:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.030345:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327385600/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030348:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829852.030350:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829852.030351:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829852.030353:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829852.030354:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829852.030355:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829852.030356:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829852.030358:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829852.030359:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829852.030360:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829852.030361:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:0.0:1787829852.030363:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829852.030364:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030365:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.030367:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327385728/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030371:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829852.030372:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829852.030373:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829852.030375:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829852.030376:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829852.030377:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829852.030379:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829852.030380:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829852.030382:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829852.030384:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829852.030385:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:0.0:1787829852.030388:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829852.030389:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030391:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829852.030394:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327385856/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829877 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829852.030401:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829852.030403:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829852.030404:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829852.030406:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829852.030407:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829852.030409:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829852.030411:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829852.030413:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:0.0:1787829852.030415:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829852.030416:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829852.030418:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:0.0:1787829852.030420:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829852.030422:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030424:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829852.030427:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829852.030429:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829852.030431:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829852.030432:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829852.540736:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829852.540743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829852.540746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829852.540752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829852.540756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829852.540759:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829852.540762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829852.540765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829852.540769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829852.732757:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298672812, at 4298672832
00000100:00000040:3.0:1787829852.732757:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298672812, at 4298672832
00000100:00000010:2.0:1787829852.732771:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00000100:00000010:3.0:1787829852.732774:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00010000:00000001:0.0:1787829852.860770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829852.860777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829852.860783:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829852.860785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829852.860788:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829852.860790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829852.860810:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829852.860813:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829852.860816:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829853.564770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829853.564775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829853.564777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829853.564781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829853.564784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829853.564786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829853.564788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829853.564790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829853.564793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829853.884768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829853.884774:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829853.884779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829853.884780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829853.884782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829853.884783:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829853.884833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829853.884836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829853.884839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829854.588754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829854.588760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829854.588764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829854.588768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829854.588772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829854.588774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829854.588776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829854.588778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829854.588781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829854.908769:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829854.908777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829854.908783:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829854.908785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829854.908790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829854.908792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829854.908855:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829854.908859:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829854.908863:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829855.612772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829855.612778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829855.612780:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829855.612784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829855.612787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829855.612790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829855.612792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829855.612794:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829855.612797:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829855.932787:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829855.932790:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829855.932793:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932798:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829855.932800:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932802:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932803:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932806:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829855.932808:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829855.932809:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829855.932811:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932813:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829855.932814:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932816:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932818:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829855.932820:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829855.932821:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829855.932862:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829855.932865:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829855.932867:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829856.636764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829856.636770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829856.636772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829856.636776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829856.636779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829856.636782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829856.636784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829856.636786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829856.636788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829856.956752:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829856.956758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829856.956763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829856.956764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829856.956767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829856.956769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829856.956822:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829856.956825:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829856.956828:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829857.148730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829857.148739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829857.148742:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829857.148745:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829857.148748:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829857.148754:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4004
00000400:00000200:2.0:1787829857.148760:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829857.148765:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829857.148768:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829857.148771:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148773:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.148775:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148777:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829857.148780:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829857.148782:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829857.148784:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829857.148787:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829857.148790:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829857.148793:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829857.148797:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148799:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.148802:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148804:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829857.148808:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829857.148812:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148815:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829857.148818:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829857.148821:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829857.148825:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.148827:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829857.148830:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829857.148839:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787829857.148843:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829857.148846:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829857.148847:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.148849:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829857.148851:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829857.148855:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829857.148861:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829857.148867:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829857.148874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829857.148882:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829857.148886:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829857.148889:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829857.148909:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829857.148912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829857.148914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829857.148915:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829857.148916:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829857.148918:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829857.148920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4004
00000400:00000200:2.0:1787829857.148923:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829857.148924:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829857.148926:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829857.148928:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148930:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.148931:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148932:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829857.148934:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829857.148936:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829857.148938:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829857.148941:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829857.148943:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829857.148946:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829857.148950:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148953:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.148955:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829857.148959:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829857.148959:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829857.148964:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829857.148967:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829857.148971:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787829857.148971:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829857.148977:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829857.148981:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829857.148983:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829857.148987:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829857.148992:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.148996:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829857.149001:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:2.0:1787829857.149002:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829857.149003:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787829857.149006:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.149009:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787829857.149009:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.149013:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829857.149016:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829857.149016:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829857.149021:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00100000:0.0:1787829857.149022:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327385984:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787829857.149025:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829857.149027:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.149030:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787829857.149030:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829857.149034:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829857.149036:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829857.149037:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.149040:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829857.149042:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829857.149047:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787829857.149048:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829857.149055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:0.0:1787829857.149056:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829857.149063:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.149066:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829857.149069:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149074:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829857.149076:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829857.149081:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000001:2.0:1787829857.149086:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829857.149091:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00080000:2.0:1787829857.149091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829857.149098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829857.149100:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:0.0:1787829857.149101:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327385984, portal 25
00000100:00080000:2.0:1787829857.149102:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829857.149106:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787829857.149108:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829857.149110:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402430400 : -116696307121216 : ffff95dd861143c0)
00000100:00080000:2.0:1787829857.149114:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4004
00000400:00000200:2.0:1787829857.149122:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:0.0:1787829857.149123:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829857.149129:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829857.149136:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829857.149141:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829857.149144:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787829857.149144:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.149148:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149151:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829857.149154:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829857.149154:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829857.149156:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829857.149159:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1787829857.149162:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000020:00000040:2.0:1787829857.149162:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829857.149165:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829857.149168:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:0.0:1787829857.149170:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327385984, offset 0
00000100:00000040:2.0:1787829857.149172:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149175:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.149176:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149177:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829857.149178:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829857.149180:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149181:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829857.149183:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829857.149185:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829857.149188:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.149189:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829857.149191:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829857.149196:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149197:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149198:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829857.149204:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787829857.149206:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829857.149209:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829857.149210:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829857.149210:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.149214:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829857.149216:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829857.149218:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149221:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000010:2.0:1787829857.149222:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829857.149229:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149231:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:2.0:1787829857.149236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829857.149239:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829857.149247:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829857.149251:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829857.149254:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149256:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149262:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40787:40787:256:4294967295] 192.168.203.116@tcp LPNI seq info [40787:40787:8:4294967295]
00000100:00000040:2.0:1787829857.149266:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829857.149274:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829857.149283:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829857.149289:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829857.149295:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1787829857.149295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829857.149298:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829857.149301:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829857.149306:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829857.149310:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4004
00000800:00000200:0.0:1787829857.149312:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829857.149317:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829857.149320:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:0.0:1787829857.149322:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00080000:2.0:1787829857.149322:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829857.149325:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149329:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829857.149330:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149332:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829857.149336:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.0:1787829857.149338:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787829857.149338:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829857.149342:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829857.149344:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000200:0.0:1787829857.149347:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000020:00000001:2.0:1787829857.149347:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829857.149349:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829857.149356:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149358:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1787829857.149361:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829857.149363:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149369:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829857.149369:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000020:00000040:2.0:1787829857.149371:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829857.149377:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000100:0.0:1787829857.149383:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829857.149383:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829857.149385:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:0.0:1787829857.149389:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000001:2.0:1787829857.149389:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829857.149394:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.149396:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829857.149398:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:2.0:1787829857.149399:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829857.149403:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.149407:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149408:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000010:2.0:1787829857.149411:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000400:00000200:0.0:1787829857.149415:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000020:00000040:2.0:1787829857.149415:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000400:00000010:0.0:1787829857.149420:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
02000000:00000001:2.0:1787829857.149421:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829857.149423:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.149426:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829857.149427:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787829857.149427:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829857.149431:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829857.149449:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000200:0.0:1787829857.149450:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829857.149453:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829857.149462:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829857.149470:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829857.149471:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829857.149476:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149480:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829857.149486:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829857.149490:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829857.149491:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4014)
00000100:00000001:0.0:1787829857.149502:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.149505:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829857.149511:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149516:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829857.149519:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298677932
00000100:00000010:0.0:1787829857.149523:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc5a0.
00000100:00000001:0.0:1787829857.149529:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829857.149533:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298677932
00000100:00000010:0.0:1787829857.149535:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc240.
00000100:00000001:0.0:1787829857.149537:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829857.149539:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829857.149541:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.149543:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829857.149546:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829857/real 1787829857]  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829857.149553:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829857.149554:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829857.149555:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829857.149558:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829857.149559:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829857.149561:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829857.149563:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829857.149569:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149574:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829857.149575:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149576:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.149578:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829857.149580:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149584:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829857.149588:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149593:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829857.149595:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149596:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.149597:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149599:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829857.149600:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829857.149602:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829857.149604:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829857.149606:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829857.149608:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829857.149613:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327385984:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829857.149617:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149618:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.149620:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861143c0 x1874671327385984/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149635:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829857.149637:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829857.149640:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829857.149643:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829857.149645:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829857.149647:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829857.149649:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829857.149651:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829857.149682:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829857.149685:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829857.149687:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861143c0.
00000100:00000001:0.0:1787829857.149691:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829857.149692:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149695:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149702:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829857.149704:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.149706:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829857.149708:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829857.149710:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829857.149715:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149722:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829857.149724:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.149726:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829857.149728:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829857.149731:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829857.149734:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386112:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829857.149738:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829857.149740:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829857.149742:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.149744:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829857.149747:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829857.149750:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.149754:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829857.149757:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829857.149761:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829857.149764:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386112, portal 10
00000100:00000001:0.0:1787829857.149766:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829857.149767:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:0.0:1787829857.149775:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149782:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829857.149784:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829857.149787:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829857.149789:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327386112, offset 0
00000400:00000200:0.0:1787829857.149794:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829857.149798:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829857.149802:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829857.149806:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829857.149808:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829857.149811:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829857.149813:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829857.149817:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40788:40788:256:4294967295] 192.168.203.116@tcp LPNI seq info [40788:40788:8:4294967295]
00000400:00000200:0.0:1787829857.149821:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829857.149829:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829857.149834:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829857.149838:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829857.149842:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829857.149846:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787829857.149851:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829857.149854:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787829857.149858:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829857.149860:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829857.149863:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829857.149867:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829857.149871:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829857.149873:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829857.149876:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829857.149880:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149889:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.149892:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149899:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.149901:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829857.149904:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829857.149908:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829857.149912:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829857.149914:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298677259
00000100:00000001:0.0:1787829857.149917:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298677932
00000100:00000001:0.0:1787829857.149919:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298677259
00000100:00000001:0.0:1787829857.149922:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298677932
00000100:00000001:0.0:1787829857.149924:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829857.149927:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.149929:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829857.149933:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829857/real 1787829857]  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829857.149942:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829857.149944:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829857.149945:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829857.149948:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829857.149950:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829857.149952:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829857.149956:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829857.149966:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149980:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829857.149982:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.149984:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.149985:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829857.149989:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.149997:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829857.150001:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150009:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829857.150012:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150013:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.150015:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150017:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829857.150019:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829857.150022:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829857.150026:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829857.150028:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829857.150032:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829857.150042:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386112:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829857.150046:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829857.150050:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150057:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829857.150058:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.150061:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829857.150063:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829857.150065:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829857.150068:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386240:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829857.150072:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829857.150074:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829857.150076:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.150077:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829857.150080:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829857.150082:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.150086:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829857.150090:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829857.150093:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829857.150096:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386240, portal 4
00000100:00000001:0.0:1787829857.150098:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829857.150099:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:0.0:1787829857.150105:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150113:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829857.150115:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829857.150117:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829857.150120:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327386240, offset 0
00000400:00000200:0.0:1787829857.150124:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829857.150128:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829857.150132:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829857.150135:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829857.150138:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829857.150140:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829857.150143:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829857.150146:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40789:40789:256:4294967295] 192.168.203.116@tcp LPNI seq info [40789:40789:8:4294967295]
00000400:00000200:0.0:1787829857.150150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829857.150157:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829857.150162:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829857.150166:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829857.150170:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829857.150173:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787829857.150178:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829857.150181:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787829857.150186:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829857.150188:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829857.150191:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829857.150194:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829857.150198:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829857.150200:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829857.150202:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829857.150206:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150214:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.150218:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150225:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.150227:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829857.150230:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829857.150233:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829857.150237:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829857.150239:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298677266
00000100:00000001:0.0:1787829857.150241:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298677932
00000100:00000001:0.0:1787829857.150243:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298677266
00000100:00000001:0.0:1787829857.150246:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298677932
00000100:00000001:0.0:1787829857.150248:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829857.150250:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.150252:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829857.150256:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829857/real 1787829857]  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829857.150265:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829857.150266:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829857.150268:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829857.150271:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829857.150272:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829857.150275:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829857.150279:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829857.150288:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150296:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829857.150297:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150299:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.150300:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829857.150305:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150312:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829857.150316:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150324:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829857.150327:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150328:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.150330:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150332:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829857.150334:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829857.150336:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829857.150339:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829857.150342:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829857.150347:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829857.150356:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386240:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829857.150360:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829857.150364:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150372:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829857.150373:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.150376:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829857.150378:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829857.150380:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829857.150383:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386368:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829857.150386:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829857.150388:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829857.150390:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829857.150392:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829857.150396:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829857.150398:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829857.150401:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829857.150403:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829857.150406:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829857.150409:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386368, portal 4
00000100:00000001:0.0:1787829857.150411:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829857.150413:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:0.0:1787829857.150419:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150426:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829857.150428:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829857.150431:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829857.150448:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327386368, offset 0
00000400:00000200:0.0:1787829857.150454:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829857.150458:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829857.150462:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829857.150465:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829857.150468:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829857.150471:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829857.150473:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829857.150476:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40790:40790:256:4294967295] 192.168.203.116@tcp LPNI seq info [40790:40790:8:4294967295]
00000400:00000200:0.0:1787829857.150481:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829857.150488:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829857.150493:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829857.150497:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829857.150500:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829857.150503:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787829857.150508:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829857.150511:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787829857.150515:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829857.150517:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829857.150520:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829857.150524:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829857.150527:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829857.150529:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829857.150531:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829857.150535:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150543:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.150547:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150554:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.150556:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829857.150559:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829857.150562:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829857.150566:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829857.150568:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298677271
00000100:00000001:0.0:1787829857.150571:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298677932
00000100:00000001:0.0:1787829857.150573:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298677271
00000100:00000001:0.0:1787829857.150576:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298677932
00000100:00000001:0.0:1787829857.150578:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829857.150580:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.150582:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829857.150586:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829857/real 1787829857]  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829857.150594:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829857.150596:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829857.150598:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829857.150600:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829857.150602:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829857.150604:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829857.150608:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829857.150616:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150623:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829857.150624:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150626:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.150628:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829857.150632:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150639:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829857.150643:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150650:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829857.150653:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150673:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829857.150674:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150677:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829857.150679:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829857.150681:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829857.150685:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829857.150688:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829857.150692:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829857.150701:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386368:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829857.150705:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150707:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.150711:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327386112/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150719:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829857.150720:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829857.150722:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829857.150725:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829857.150727:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829857.150728:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829857.150731:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829857.150733:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829857.150735:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829857.150737:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829857.150738:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:0.0:1787829857.150742:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829857.150743:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150745:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.150748:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327386240/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150761:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829857.150763:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829857.150764:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829857.150766:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829857.150768:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829857.150769:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829857.150771:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829857.150772:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829857.150774:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829857.150775:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829857.150777:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:0.0:1787829857.150779:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829857.150780:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150782:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829857.150785:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327386368/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829857.150792:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829857.150793:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829857.150795:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829857.150797:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829857.150799:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829857.150800:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829857.150802:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829857.150804:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829857.150806:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829857.150807:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829857.150808:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:0.0:1787829857.150810:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829857.150811:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150813:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829857.150817:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829857.150819:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829857.150821:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829857.150822:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829857.660758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829857.660764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829857.660766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829857.660769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829857.660772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829857.660775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829857.660777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829857.660779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829857.660781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829857.852802:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298677932, at 4298677952
00000100:00000040:3.0:1787829857.852803:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298677932, at 4298677952
00000100:00000010:2.0:1787829857.852820:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000010:3.0:1787829857.852823:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00010000:00000001:0.0:1787829857.980790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829857.980798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829857.980803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829857.980804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829857.980808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829857.980809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829857.980881:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829857.980887:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829857.980891:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829858.684758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829858.684764:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829858.684765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829858.684769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829858.684772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829858.684775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829858.684777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829858.684779:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829858.684781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829859.004782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829859.004788:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829859.004793:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829859.004794:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829859.004796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829859.004798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829859.004821:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829859.004823:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829859.004825:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829859.708792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829859.708800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829859.708802:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829859.708808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829859.708812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829859.708816:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829859.708819:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829859.708821:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829859.708824:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829860.028787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829860.028795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829860.028802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829860.028804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829860.028808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829860.028809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829860.028861:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829860.028864:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829860.028868:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829860.732759:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829860.732765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829860.732767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829860.732771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829860.732774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829860.732776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829860.732778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829860.732781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829860.732783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829861.052761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829861.052768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829861.052775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829861.052777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829861.052781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829861.052784:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829861.052786:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829861.052789:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829861.052793:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829861.052795:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829861.052797:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829861.052800:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829861.052848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829861.052852:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829861.052856:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829861.756740:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829861.756748:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829861.756750:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829861.756756:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829861.756760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829861.756765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829861.756767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829861.756770:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829861.756774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829862.076788:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829862.076795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829862.076802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829862.076804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829862.076807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829862.076809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829862.076863:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829862.076867:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829862.076871:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829862.268773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829862.268781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829862.268783:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829862.268787:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829862.268790:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829862.268796:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4009
00000400:00000200:1.0:1787829862.268803:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829862.268807:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829862.268810:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829862.268812:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268814:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829862.268816:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268818:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829862.268821:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829862.268822:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829862.268824:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829862.268826:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829862.268828:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829862.268829:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829862.268831:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268832:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829862.268834:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268836:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829862.268838:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829862.268840:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268842:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829862.268844:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829862.268847:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829862.268849:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.268850:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829862.268852:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829862.268860:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829862.268862:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829862.268865:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829862.268866:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.268868:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829862.268870:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829862.268874:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829862.268880:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829862.268885:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829862.268891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829862.268899:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829862.268903:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829862.268913:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829862.268930:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829862.268933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829862.268935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829862.268936:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829862.268937:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829862.268939:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829862.268941:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4009
00000400:00000200:1.0:1787829862.268944:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829862.268946:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829862.268948:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829862.268950:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268951:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829862.268953:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268954:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829862.268955:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829862.268956:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829862.268957:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829862.268958:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829862.268959:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829862.268961:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829862.268963:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268964:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829862.268965:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268967:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829862.268968:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829862.268970:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829862.268971:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829862.268972:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829862.268976:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829862.268978:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.268979:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829862.268982:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829862.268985:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.268986:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829862.268989:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829862.268990:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829862.268992:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829862.268994:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.268995:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829862.268996:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829862.268998:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269001:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000010:1.0:1787829862.269001:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:0.0:1787829862.269007:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829862.269007:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269011:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00100000:1.0:1787829862.269014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829862.269020:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829862.269022:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829862.269025:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829862.269034:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269037:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829862.269041:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269047:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787829862.269050:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269052:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787829862.269055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829862.269057:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829862.269063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00100000:0.0:1787829862.269066:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327386496:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829862.269067:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829862.269071:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829862.269073:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829862.269076:0:36:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:0.0:1787829862.269079:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:1.0:1787829862.269083:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4009
02000000:00000001:0.0:1787829862.269084:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269090:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:1.0:1787829862.269091:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829862.269096:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
02000000:00000010:0.0:1787829862.269097:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
00000100:00080000:1.0:1787829862.269099:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000001:0.0:1787829862.269100:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829862.269107:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269108:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829862.269114:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829862.269115:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000040:1.0:1787829862.269117:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.269120:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829862.269122:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829862.269123:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829862.269124:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1787829862.269125:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000020:00000040:1.0:1787829862.269126:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829862.269131:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829862.269133:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:0.0:1787829862.269135:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386496, portal 25
00000100:00000040:1.0:1787829862.269136:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.269139:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829862.269142:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:1.0:1787829862.269143:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829862.269147:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000001:1.0:1787829862.269148:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829862.269154:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829862.269157:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829862.269160:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829862.269162:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829862.269163:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829862.269165:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829862.269170:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.269174:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829862.269176:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829862.269179:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:1.0:1787829862.269180:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829862.269184:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829862.269184:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829862.269188:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829862.269191:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327386496, offset 0
00000100:00000010:1.0:1787829862.269196:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000400:00000200:0.0:1787829862.269201:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:1.0:1787829862.269201:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829862.269206:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829862.269207:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.269208:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829862.269209:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829862.269212:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829862.269213:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829862.269220:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000010:1.0:1787829862.269220:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829862.269227:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829862.269228:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829862.269234:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00100000:1.0:1787829862.269238:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829862.269239:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829862.269244:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269252:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40791:40791:256:4294967295] 192.168.203.116@tcp LPNI seq info [40791:40791:8:4294967295]
00000400:00000200:0.0:1787829862.269259:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829862.269262:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829862.269266:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829862.269272:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787829862.269275:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829862.269281:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.269289:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829862.269289:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829862.269294:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1787829862.269298:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829862.269299:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829862.269303:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.0:1787829862.269305:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00080000:1.0:1787829862.269305:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829862.269312:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1787829862.269315:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829862.269317:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4009
00000800:00000200:0.1:1787829862.269319:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000400:00000200:1.0:1787829862.269324:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000100:0.0:1787829862.269328:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829862.269331:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:0.0:1787829862.269333:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00080000:1.0:1787829862.269334:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829862.269338:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269339:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787829862.269340:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829862.269345:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269348:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787829862.269349:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829862.269352:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829862.269353:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000020:00000040:1.0:1787829862.269354:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829862.269358:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1787829862.269359:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000020:00000040:1.0:1787829862.269360:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829862.269363:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829862.269364:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:1.0:1787829862.269365:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829862.269371:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829862.269372:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829862.269375:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829862.269379:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829862.269383:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829862.269386:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829862.269390:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829862.269391:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829862.269392:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829862.269395:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829862.269412:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829862.269412:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829862.269417:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.269421:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829862.269423:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829862.269425:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829862.269426:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829862.269431:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829862.269433:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829862.269436:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829862.269436:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829862.269440:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269441:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829862.269442:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829862.269445:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829862.269446:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269448:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829862.269449:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000400:00000200:0.0:1787829862.269455:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829862.269455:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269460:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298683052
00000100:00100000:1.0:1787829862.269463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:0.0:1787829862.269466:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc960.
00000100:00000001:1.0:1787829862.269472:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829862.269474:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269476:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787829862.269477:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269482:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298683052
00000100:00000001:1.0:1787829862.269485:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829862.269487:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc00.
00000100:00000001:0.0:1787829862.269491:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000040:1.0:1787829862.269491:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4020)
00000100:00000001:0.0:1787829862.269505:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829862.269507:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269512:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829862.269517:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829862/real 1787829862]  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829862.269524:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829862.269526:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829862.269528:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829862.269531:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829862.269533:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829862.269536:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829862.269539:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829862.269545:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269549:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829862.269550:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269551:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.269553:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829862.269555:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269562:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829862.269566:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269570:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829862.269572:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269574:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.269575:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269577:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829862.269578:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829862.269580:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829862.269583:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829862.269584:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829862.269587:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829862.269592:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327386496:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829862.269596:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269597:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.269599:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327386496/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269604:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829862.269606:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829862.269608:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829862.269610:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829862.269612:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829862.269613:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829862.269614:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829862.269616:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829862.269619:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829862.269620:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829862.269621:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829862.269624:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829862.269624:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269626:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269631:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829862.269632:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269634:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829862.269635:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829862.269636:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829862.269639:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269644:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829862.269645:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269646:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829862.269647:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829862.269649:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829862.269651:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386624:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829862.269653:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829862.269679:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829862.269681:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269683:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829862.269685:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829862.269687:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829862.269689:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829862.269691:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829862.269694:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829862.269696:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386624, portal 10
00000100:00000001:0.0:1787829862.269697:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829862.269698:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829862.269703:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269708:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829862.269709:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829862.269711:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829862.269712:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327386624, offset 0
00000400:00000200:0.0:1787829862.269715:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269717:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829862.269720:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829862.269722:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829862.269724:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829862.269726:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829862.269728:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269730:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40792:40792:256:4294967295] 192.168.203.116@tcp LPNI seq info [40792:40792:8:4294967295]
00000400:00000200:0.0:1787829862.269733:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829862.269737:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829862.269740:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.269742:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829862.269745:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829862.269747:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829862.269750:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829862.269752:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787829862.269754:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.269755:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829862.269758:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829862.269760:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829862.269762:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829862.269763:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829862.269765:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829862.269767:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269772:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.269774:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269779:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269780:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829862.269782:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829862.269784:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829862.269786:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829862.269788:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298682379
00000100:00000001:0.0:1787829862.269790:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298683052
00000100:00000001:0.0:1787829862.269791:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298682379
00000100:00000001:0.0:1787829862.269793:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298683052
00000100:00000001:0.0:1787829862.269795:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829862.269796:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269797:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829862.269800:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829862/real 1787829862]  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829862.269804:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829862.269805:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829862.269806:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829862.269807:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829862.269808:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829862.269810:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829862.269812:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829862.269817:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269821:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829862.269822:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269823:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.269824:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829862.269826:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269830:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829862.269832:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269836:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829862.269838:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269839:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.269840:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.269841:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829862.269842:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829862.269845:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829862.269847:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829862.269848:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829862.269851:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829862.269855:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386624:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829862.269858:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829862.269860:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269864:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829862.269865:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269866:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829862.269867:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829862.269868:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829862.269870:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386752:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829862.269872:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829862.269873:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829862.269874:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.269876:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829862.269878:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829862.269879:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829862.269881:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829862.269883:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829862.269885:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829862.269887:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386752, portal 4
00000100:00000001:0.0:1787829862.269888:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829862.269889:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829862.269893:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269901:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829862.269903:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829862.269904:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829862.269906:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327386752, offset 0
00000400:00000200:0.0:1787829862.269908:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269910:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829862.269912:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829862.269914:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829862.269915:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829862.269917:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829862.269918:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.269920:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40793:40793:256:4294967295] 192.168.203.116@tcp LPNI seq info [40793:40793:8:4294967295]
00000400:00000200:0.0:1787829862.269923:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829862.269927:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829862.269930:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.269932:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829862.269934:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829862.269936:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829862.269939:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829862.269940:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787829862.269943:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.269944:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829862.269946:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829862.269948:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829862.269950:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829862.269951:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829862.269953:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829862.269955:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269960:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.269961:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.269970:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269971:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829862.269973:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829862.269975:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829862.269977:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829862.269978:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298682386
00000100:00000001:0.0:1787829862.269980:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298683052
00000100:00000001:0.0:1787829862.269982:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298682386
00000100:00000001:0.0:1787829862.269983:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298683052
00000100:00000001:0.0:1787829862.269984:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829862.269986:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.269987:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829862.269989:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829862/real 1787829862]  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829862.269993:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829862.269994:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829862.269995:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829862.269996:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829862.269997:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829862.269998:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829862.270000:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829862.270005:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270009:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829862.270010:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270011:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.270012:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829862.270014:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270019:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829862.270022:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270027:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829862.270029:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270030:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.270031:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270033:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829862.270035:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829862.270037:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829862.270041:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829862.270043:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829862.270047:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829862.270056:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386752:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829862.270060:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829862.270064:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270071:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829862.270073:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.270075:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829862.270076:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829862.270079:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829862.270082:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386880:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829862.270085:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829862.270087:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829862.270088:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829862.270090:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829862.270092:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829862.270093:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829862.270096:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829862.270098:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829862.270100:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829862.270102:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327386880, portal 4
00000100:00000001:0.0:1787829862.270103:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829862.270104:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:0.0:1787829862.270108:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270113:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829862.270114:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829862.270116:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829862.270118:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327386880, offset 0
00000400:00000200:0.0:1787829862.270123:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829862.270127:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829862.270130:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829862.270133:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829862.270136:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829862.270138:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829862.270141:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829862.270144:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40794:40794:256:4294967295] 192.168.203.116@tcp LPNI seq info [40794:40794:8:4294967295]
00000400:00000200:0.0:1787829862.270148:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829862.270155:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829862.270160:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.270163:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829862.270168:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829862.270170:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787829862.270173:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829862.270175:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787829862.270177:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829862.270179:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829862.270181:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829862.270183:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829862.270186:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829862.270187:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829862.270188:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829862.270191:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270197:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.270199:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270203:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.270204:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829862.270206:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829862.270208:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829862.270211:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829862.270212:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298682391
00000100:00000001:0.0:1787829862.270214:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298683052
00000100:00000001:0.0:1787829862.270215:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298682391
00000100:00000001:0.0:1787829862.270217:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298683052
00000100:00000001:0.0:1787829862.270218:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829862.270219:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.270220:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829862.270223:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829862/real 1787829862]  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829862.270228:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829862.270229:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829862.270230:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829862.270231:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829862.270233:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829862.270234:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829862.270236:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829862.270242:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270246:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829862.270247:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270248:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.270249:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829862.270251:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270255:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829862.270257:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270261:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829862.270263:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270264:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829862.270265:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270266:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829862.270267:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829862.270269:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829862.270271:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829862.270272:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829862.270274:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829862.270279:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327386880:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829862.270281:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270282:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.270285:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327386624/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270289:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829862.270290:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829862.270292:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829862.270293:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829862.270294:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829862.270295:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829862.270296:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829862.270298:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829862.270300:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829862.270300:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829862.270302:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829862.270303:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829862.270304:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270306:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.270308:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327386752/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270312:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829862.270312:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829862.270313:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829862.270315:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829862.270315:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829862.270316:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829862.270317:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829862.270318:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:0.0:1787829862.270319:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829862.270320:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829862.270320:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829862.270322:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829862.270322:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270323:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829862.270325:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327386880/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829862.270329:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829862.270330:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829862.270331:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829862.270332:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829862.270333:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829862.270334:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829862.270335:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829862.270336:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829862.270337:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829862.270338:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829862.270339:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829862.270340:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829862.270341:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270342:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829862.270344:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829862.270345:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829862.270346:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829862.270347:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829862.780757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829862.780763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829862.780765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829862.780769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829862.780772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829862.780775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829862.780777:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829862.780778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829862.780780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829862.972770:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298683052, at 4298683072
00000100:00000040:3.0:1787829862.972770:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298683052, at 4298683072
00000100:00000010:2.0:1787829862.972787:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00000100:00000010:3.0:1787829862.972790:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00010000:00000001:0.0:1787829863.100752:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829863.100757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829863.100761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829863.100762:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829863.100765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829863.100766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829863.100808:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829863.100811:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829863.100814:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829863.804760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829863.804765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829863.804766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829863.804770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829863.804773:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829863.804776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829863.804779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829863.804781:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829863.804784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829864.124802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829864.124811:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829864.124819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829864.124821:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829864.124826:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829864.124828:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829864.124893:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829864.124898:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829864.124902:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829864.828766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829864.828772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829864.828773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829864.828777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829864.828779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829864.828782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829864.828784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829864.828786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829864.828788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829865.148816:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829865.148824:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829865.148829:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829865.148830:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829865.148833:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829865.148835:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829865.148868:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829865.148871:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829865.148874:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829865.852730:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829865.852735:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829865.852736:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829865.852740:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829865.852743:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829865.852746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829865.852748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829865.852750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829865.852752:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172756:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829866.172764:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829866.172766:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829866.172768:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172772:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829866.172774:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172776:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172778:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829866.172781:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829866.172782:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829866.172783:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172784:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829866.172785:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172786:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172787:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829866.172789:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829866.172790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829866.172839:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829866.172844:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829866.172847:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829866.876718:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829866.876724:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829866.876726:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829866.876729:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829866.876732:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829866.876735:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829866.876737:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829866.876739:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829866.876741:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829867.196733:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829867.196738:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829867.196743:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829867.196744:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829867.196747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829867.196748:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829867.196818:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829867.196823:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829867.196827:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829867.388737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829867.388744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829867.388746:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829867.388750:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829867.388754:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829867.388759:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4015
00000400:00000200:1.0:1787829867.388767:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829867.388772:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829867.388775:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829867.388777:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388779:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829867.388782:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388783:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829867.388786:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829867.388788:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829867.388790:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829867.388793:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829867.388794:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829867.388796:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829867.388798:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388799:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829867.388800:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388801:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829867.388804:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829867.388812:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388814:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829867.388816:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829867.388819:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829867.388821:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.388823:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829867.388824:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829867.388832:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829867.388834:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829867.388837:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829867.388838:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.388845:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829867.388848:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829867.388851:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829867.388859:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829867.388864:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829867.388871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829867.388878:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829867.388883:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829867.388886:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829867.388912:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829867.388916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829867.388919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829867.388921:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829867.388923:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829867.388926:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829867.388929:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4015
00000400:00000200:1.0:1787829867.388933:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829867.388937:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829867.388940:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829867.388944:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388947:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829867.388950:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.388952:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829867.388953:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787829867.388954:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829867.388958:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829867.388959:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787829867.388960:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829867.388963:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787829867.388965:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829867.388969:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829867.388972:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829867.388974:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829867.388985:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829867.388986:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829867.388989:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.388992:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829867.388993:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829867.388996:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787829867.388997:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829867.388999:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389003:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:0.0:1787829867.389006:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327387008:192.168.203.116@tcp:250:kworker.0
00000020:00000040:1.0:1787829867.389010:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829867.389012:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:1.0:1787829867.389016:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:0.0:1787829867.389017:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829867.389021:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389023:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.389024:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787829867.389027:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:0.0:1787829867.389031:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829867.389033:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829867.389034:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829867.389037:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389041:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829867.389043:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:1.0:1787829867.389045:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1787829867.389047:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000008:00000001:1.0:1787829867.389049:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389052:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829867.389054:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000010:1.0:1787829867.389056:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829867.389060:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000200:0.0:1787829867.389062:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387008, portal 25
02000000:00000001:1.0:1787829867.389062:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829867.389065:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389066:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829867.389068:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829867.389069:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
02000000:00000001:1.0:1787829867.389071:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829867.389074:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829867.389077:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787829867.389080:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829867.389086:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389088:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829867.389092:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00100000:1.0:1787829867.389095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829867.389096:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829867.389102:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327387008, offset 0
00000100:00000001:1.0:1787829867.389107:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829867.389110:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829867.389113:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829867.389117:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829867.389124:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829867.389133:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829867.389134:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:1.0:1787829867.389136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829867.389140:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1787829867.389141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829867.389144:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829867.389145:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829867.389146:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:1.0:1787829867.389151:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829867.389154:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1787829867.389158:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4015
00000400:00000200:0.0:1787829867.389161:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829867.389165:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40795:40795:256:4294967295] 192.168.203.116@tcp LPNI seq info [40795:40795:8:4294967295]
00000400:00000200:0.0:1787829867.389168:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829867.389174:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829867.389184:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829867.389185:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1787829867.389188:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000800:00000200:0.0:1787829867.389194:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829867.389194:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389200:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:0.0:1787829867.389201:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000040:1.0:1787829867.389202:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829867.389208:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829867.389214:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1787829867.389214:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829867.389218:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829867.389222:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:0.1:1787829867.389222:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829867.389225:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.1:1787829867.389227:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000020:00000040:1.0:1787829867.389229:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000100:0.0:1787829867.389234:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:1.0:1787829867.389234:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829867.389239:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:0.0:1787829867.389240:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00000040:1.0:1787829867.389247:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829867.389248:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787829867.389249:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829867.389255:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829867.389257:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787829867.389259:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829867.389262:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000020:00000040:1.0:1787829867.389262:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829867.389266:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389268:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829867.389269:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829867.389271:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829867.389272:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829867.389275:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:1.0:1787829867.389276:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829867.389278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829867.389279:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389283:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389286:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:1.0:1787829867.389287:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000020:00000040:1.0:1787829867.389291:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000200:0.0:1787829867.389292:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829867.389293:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829867.389296:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389300:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829867.389303:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829867.389306:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829867.389311:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000001:0.0:1787829867.389314:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829867.389315:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829867.389324:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829867.389325:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829867.389347:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829867.389351:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389356:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829867.389357:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389375:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829867.389382:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389385:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829867.389385:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829867.389391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829867.389392:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829867.389395:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829867.389396:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829867.389399:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829867.389402:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298688172
00000100:00080000:1.0:1787829867.389403:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4015
00000100:00000010:0.0:1787829867.389411:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc60.
00000400:00000200:1.0:1787829867.389416:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829867.389419:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829867.389421:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:1.0:1787829867.389422:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829867.389427:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389428:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829867.389430:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298688172
00000100:00000040:1.0:1787829867.389431:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389436:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000010:0.0:1787829867.389438:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc3c0.
00000020:00000001:1.0:1787829867.389438:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829867.389441:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829867.389444:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829867.389446:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829867.389447:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000001:1.0:1787829867.389448:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829867.389453:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829867.389455:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389456:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829867.389457:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389459:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829867.389460:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829867.389462:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829867.389463:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389465:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829867.389465:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829867.389469:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829867.389472:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829867.389472:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389478:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829867.389479:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000008:00100000:1.0:1787829867.389481:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829867.389484:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389487:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829867.389489:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867]  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787829867.389492:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829867.389496:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829867.389501:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829867.389503:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829867.389508:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829867.389511:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829867.389513:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829867.389514:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000010:1.0:1787829867.389517:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000400:00000001:0.0:1787829867.389518:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1787829867.389522:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829867.389523:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829867.389530:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000100:00100000:1.0:1787829867.389530:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829867.389536:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829867.389543:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787829867.389543:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829867.389546:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829867.389555:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829867.389556:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829867.389573:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829867.389579:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4025)
00000100:00000200:0.0:1787829867.389585:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389594:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829867.389597:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389600:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.389602:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829867.389607:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389615:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829867.389619:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389624:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829867.389626:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389627:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.389628:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389630:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829867.389631:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829867.389633:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829867.389635:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829867.389637:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829867.389640:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829867.389646:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327387008:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829867.389649:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389651:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.389653:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327387008/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389690:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829867.389692:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829867.389694:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829867.389696:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829867.389698:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829867.389699:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829867.389700:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829867.389702:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829867.389705:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829867.389706:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829867.389707:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829867.389710:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829867.389710:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389712:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389717:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829867.389719:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389721:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829867.389722:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829867.389723:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829867.389726:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389731:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829867.389732:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.389733:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829867.389734:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829867.389736:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829867.389738:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387136:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829867.389740:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829867.389742:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829867.389743:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.389744:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829867.389747:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829867.389748:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829867.389751:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829867.389753:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829867.389756:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829867.389758:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387136, portal 10
00000100:00000001:0.0:1787829867.389759:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829867.389760:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829867.389765:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389769:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829867.389771:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829867.389772:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829867.389774:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327387136, offset 0
00000400:00000200:0.0:1787829867.389777:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829867.389780:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829867.389783:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829867.389785:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829867.389787:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829867.389789:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829867.389790:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829867.389792:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40796:40796:256:4294967295] 192.168.203.116@tcp LPNI seq info [40796:40796:8:4294967295]
00000400:00000200:0.0:1787829867.389795:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829867.389800:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829867.389803:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829867.389805:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829867.389808:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829867.389810:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829867.389813:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829867.389814:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829867.389828:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829867.389830:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829867.389832:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829867.389835:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829867.389837:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829867.389839:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829867.389847:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829867.389851:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389856:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.389858:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389863:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389864:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829867.389867:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829867.389870:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829867.389872:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829867.389873:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298687499
00000100:00000001:0.0:1787829867.389875:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298688172
00000100:00000001:0.0:1787829867.389877:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298687499
00000100:00000001:0.0:1787829867.389878:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298688172
00000100:00000001:0.0:1787829867.389880:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829867.389881:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.389882:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829867.389885:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867]  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829867.389890:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829867.389890:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829867.389892:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829867.389893:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829867.389894:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829867.389895:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829867.389897:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829867.389902:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389911:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829867.389912:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389913:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.389914:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829867.389916:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389920:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829867.389923:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389927:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829867.389928:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389929:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.389930:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.389931:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829867.389932:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829867.389934:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829867.389936:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829867.389937:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829867.389940:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829867.389944:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387136:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829867.389947:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829867.389949:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389953:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829867.389954:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.389956:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829867.389958:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829867.389959:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829867.389961:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387264:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829867.389963:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829867.389964:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829867.389965:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.389966:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829867.389968:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829867.389969:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829867.389972:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829867.389973:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829867.389976:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829867.389977:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387264, portal 4
00000100:00000001:0.0:1787829867.389979:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829867.389980:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829867.389983:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.389987:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829867.389989:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829867.389990:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829867.389992:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327387264, offset 0
00000400:00000200:0.0:1787829867.389994:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829867.389997:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829867.389999:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829867.390000:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829867.390002:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829867.390004:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829867.390005:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829867.390007:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40797:40797:256:4294967295] 192.168.203.116@tcp LPNI seq info [40797:40797:8:4294967295]
00000400:00000200:0.0:1787829867.390010:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829867.390014:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829867.390018:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829867.390020:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829867.390022:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829867.390024:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829867.390027:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829867.390029:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829867.390031:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829867.390032:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829867.390034:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829867.390036:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829867.390038:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829867.390040:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829867.390041:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829867.390043:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390048:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.390050:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390054:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.390055:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829867.390057:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829867.390058:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829867.390060:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829867.390061:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298687506
00000100:00000001:0.0:1787829867.390063:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298688172
00000100:00000001:0.0:1787829867.390064:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298687506
00000100:00000001:0.0:1787829867.390066:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298688172
00000100:00000001:0.0:1787829867.390067:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829867.390068:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.390069:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829867.390071:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867]  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829867.390077:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829867.390078:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829867.390079:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829867.390080:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829867.390081:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829867.390082:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829867.390084:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829867.390089:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390093:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829867.390093:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390094:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.390095:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829867.390097:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390101:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829867.390103:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390108:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829867.390110:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390111:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.390111:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390112:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829867.390113:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829867.390115:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829867.390117:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829867.390118:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829867.390121:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829867.390126:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387264:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829867.390128:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829867.390130:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390134:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829867.390135:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.390136:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829867.390137:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829867.390138:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829867.390140:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387392:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829867.390142:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829867.390143:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829867.390144:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829867.390145:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829867.390147:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829867.390148:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829867.390149:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829867.390151:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829867.390152:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829867.390154:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387392, portal 4
00000100:00000001:0.0:1787829867.390155:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829867.390156:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:0.0:1787829867.390160:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390164:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829867.390165:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829867.390167:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829867.390168:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327387392, offset 0
00000400:00000200:0.0:1787829867.390171:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829867.390173:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829867.390175:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829867.390176:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829867.390178:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829867.390180:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829867.390182:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829867.390184:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40798:40798:256:4294967295] 192.168.203.116@tcp LPNI seq info [40798:40798:8:4294967295]
00000400:00000200:0.0:1787829867.390186:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829867.390190:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829867.390192:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829867.390194:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829867.390196:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829867.390198:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829867.390201:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829867.390202:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829867.390205:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829867.390206:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829867.390208:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829867.390210:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829867.390212:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829867.390213:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829867.390214:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829867.390216:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390221:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.390222:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390226:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.390227:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829867.390229:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829867.390231:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829867.390233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829867.390234:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298687511
00000100:00000001:0.0:1787829867.390236:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298688172
00000100:00000001:0.0:1787829867.390238:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298687511
00000100:00000001:0.0:1787829867.390239:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298688172
00000100:00000001:0.0:1787829867.390240:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829867.390241:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.390242:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829867.390245:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867]  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829867.390249:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829867.390250:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829867.390251:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829867.390252:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829867.390253:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829867.390254:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829867.390256:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829867.390262:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390265:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829867.390266:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390267:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.390268:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829867.390270:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390274:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829867.390276:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390280:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829867.390282:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390283:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829867.390284:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390285:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829867.390287:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829867.390288:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829867.390290:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829867.390291:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829867.390293:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829867.390298:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387392:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829867.390300:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390301:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.390303:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327387136/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390307:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829867.390308:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829867.390309:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829867.390310:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829867.390311:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829867.390312:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829867.390313:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829867.390314:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829867.390316:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829867.390317:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829867.390318:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829867.390320:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829867.390321:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390322:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.390324:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327387264/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390332:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829867.390333:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829867.390334:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829867.390336:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829867.390337:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829867.390337:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829867.390338:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829867.390339:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829867.390340:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829867.390341:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829867.390342:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829867.390343:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829867.390344:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390344:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829867.390346:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327387392/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829867.390350:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829867.390351:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829867.390352:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829867.390353:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829867.390353:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829867.390354:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829867.390355:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829867.390356:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829867.390357:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829867.390358:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829867.390359:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829867.390360:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829867.390375:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390378:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829867.390381:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829867.390382:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829867.390383:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829867.390384:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829867.900725:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829867.900733:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829867.900735:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829867.900741:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829867.900745:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829867.900749:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829867.900759:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829867.900761:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829867.900764:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829868.092759:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298688172, at 4298688192
00000100:00000040:3.0:1787829868.092759:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298688172, at 4298688192
00000100:00000010:3.0:1787829868.092774:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00000100:00000010:2.0:1787829868.092777:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00010000:00000001:0.0:1787829868.220790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829868.220797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829868.220803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829868.220804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829868.220808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829868.220810:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829868.220829:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829868.220831:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829868.220833:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829868.924757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829868.924765:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829868.924767:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829868.924773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829868.924776:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829868.924779:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829868.924782:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829868.924785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829868.924788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829869.244759:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829869.244766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829869.244773:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829869.244774:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829869.244779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829869.244780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829869.244849:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829869.244854:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829869.244858:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829869.948725:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829869.948731:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829869.948733:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829869.948737:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829869.948740:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829869.948743:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829869.948744:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829869.948746:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829869.948748:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829870.268768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829870.268773:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829870.268777:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829870.268778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829870.268781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829870.268784:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829870.268786:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829870.268788:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829870.268791:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829870.268793:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829870.268796:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829870.268798:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829870.268858:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829870.268862:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829870.268865:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829870.972785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829870.972792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829870.972795:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829870.972800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829870.972804:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829870.972808:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829870.972811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829870.972814:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829870.972817:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829871.292762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829871.292768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829871.292772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829871.292773:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829871.292775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829871.292776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829871.292792:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829871.292794:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829871.292796:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829871.996704:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829871.996710:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829871.996712:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829871.996715:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829871.996718:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829871.996720:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829871.996722:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829871.996724:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829871.996727:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829872.316760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829872.316767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829872.316770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829872.316772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829872.316775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829872.316776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829872.316833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829872.316836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829872.316839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829872.508768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829872.508777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829872.508779:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829872.508782:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829872.508785:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829872.508792:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4020
00000400:00000200:0.0:1787829872.508800:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829872.508805:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829872.508808:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829872.508810:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508812:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.508814:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508816:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829872.508819:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829872.508820:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829872.508823:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829872.508825:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829872.508827:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829872.508828:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829872.508830:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508832:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.508833:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508835:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829872.508837:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829872.508839:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508841:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829872.508843:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829872.508846:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829872.508848:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829872.508850:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829872.508852:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829872.508860:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716c000.
00000020:00000040:0.0:1787829872.508862:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829872.508865:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829872.508866:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829872.508869:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829872.508871:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829872.508874:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829872.508880:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533800.
00000100:00000001:0.0:1787829872.508885:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829872.508892:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829872.508900:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829872.508903:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829872.508907:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cdd0ad0f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.508925:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829872.508927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829872.508930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829872.508931:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829872.508932:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829872.508934:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829872.508935:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4020
00000400:00000200:0.0:1787829872.508938:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829872.508941:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829872.508942:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829872.508945:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508947:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.508949:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508952:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829872.508953:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829872.508954:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829872.508956:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787829872.508956:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787829872.508960:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:1.0:1787829872.508961:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787829872.508962:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829872.508966:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787829872.508967:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829872.508971:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508974:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829872.508976:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829872.508977:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.508981:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829872.508985:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:0.0:1787829872.508991:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829872.508994:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829872.508995:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.508997:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829872.508998:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829872.509004:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829872.509006:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829872.509007:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829872.509009:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829872.509013:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327387520:192.168.203.116@tcp:250:kworker.0
02000000:00000001:0.0:1787829872.509019:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509020:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829872.509024:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787829872.509025:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829872.509027:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.509030:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829872.509035:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
00000008:00100000:0.0:1787829872.509036:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829872.509037:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829872.509043:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829872.509045:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829872.509047:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829872.509049:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000100:00000010:0.0:1787829872.509054:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f0c0.
00000020:00000040:0.0:1787829872.509060:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:1.0:1787829872.509062:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
02000000:00000001:0.0:1787829872.509063:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829872.509067:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829872.509070:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387520, portal 25
00000100:00000001:0.0:1787829872.509071:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787829872.509075:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787829872.509076:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787829872.509078:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
02000000:00000001:0.0:1787829872.509081:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829872.509084:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829872.509089:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533400.
00000100:00000001:1.0:1787829872.509091:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829872.509095:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829872.509097:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829872.509098:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829872.509103:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327387520, offset 0
00000100:00100000:0.0:1787829872.509107:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829872.509112:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829872.509122:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.509132:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:0.0:1787829872.509139:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829872.509142:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787829872.509143:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829872.509150:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003ce953a3] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829872.509152:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829872.509160:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787829872.509162:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829872.509166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829872.509168:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:0.0:1787829872.509170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829872.509173:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787829872.509174:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40799:40799:256:4294967295] 192.168.203.116@tcp LPNI seq info [40799:40799:8:4294967295]
00000100:00080000:0.0:1787829872.509175:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829872.509182:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829872.509183:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829872.509186:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4020
00000400:00000200:0.0:1787829872.509192:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829872.509200:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:1.0:1787829872.509201:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787829872.509205:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829872.509211:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829872.509211:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829872.509216:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.509218:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:1.0:1787829872.509218:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00000001:0.0:1787829872.509221:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829872.509224:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829872.509225:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000800:00000200:1.0:1787829872.509228:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:0.0:1787829872.509229:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829872.509233:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:1.0:1787829872.509234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787829872.509236:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:0.0:1787829872.509240:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:1.1:1787829872.509244:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:0.0:1787829872.509247:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:1.1:1787829872.509248:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787829872.509249:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.509254:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000100:1.0:1787829872.509258:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829872.509259:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000010:1.0:1787829872.509261:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000020:00000040:0.0:1787829872.509262:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:0.0:1787829872.509267:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:1.0:1787829872.509267:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787829872.509272:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829872.509274:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787829872.509275:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829872.509280:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
02000000:00000001:0.0:1787829872.509281:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1787829872.509284:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
02000000:00000001:0.0:1787829872.509285:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829872.509289:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829872.509290:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:0.0:1787829872.509291:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829872.509295:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829872.509296:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.509297:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829872.509303:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716e580.
00000020:00000040:0.0:1787829872.509308:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:1.0:1787829872.509310:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787829872.509311:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829872.509315:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829872.509316:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.509317:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829872.509320:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829872.509323:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829872.509343:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530000.
00000100:00000001:1.0:1787829872.509345:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829872.509348:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509348:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829872.509354:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829872.509356:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829872.509362:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829872.509366:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298693292
00000100:00000001:0.0:1787829872.509371:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829872.509374:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829872.509374:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63ea0.
00000100:00000040:0.0:1787829872.509379:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc9f2c41] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509382:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829872.509389:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509390:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298693292
00000100:00000010:1.0:1787829872.509395:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e635a0.
00000100:00080000:0.0:1787829872.509396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829872.509398:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:0.0:1787829872.509404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829872.509406:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829872.509408:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829872.509408:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829872.509415:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829872.509416:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787829872.509419:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1787829872.509423:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4020
00000100:00100000:1.0:1787829872.509429:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872]  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829872.509432:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829872.509438:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829872.509442:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829872.509448:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000001:1.0:1787829872.509450:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787829872.509451:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.509455:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000001:1.0:1787829872.509455:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829872.509460:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000100:00000001:0.0:1787829872.509461:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829872.509466:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829872.509467:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000200:1.0:1787829872.509467:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000020:00000001:0.0:1787829872.509469:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787829872.509471:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000020:00000040:0.0:1787829872.509472:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829872.509474:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:0.0:1787829872.509475:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829872.509478:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:1.0:1787829872.509480:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829872.509483:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.509488:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829872.509492:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829872.509498:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:1.0:1787829872.509501:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829872.509502:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829872.509509:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829872.509515:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829872.509516:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509518:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829872.509519:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829872.509522:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1787829872.509523:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829872.509524:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829872.509530:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829872.509533:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.509535:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829872.509539:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829872.509543:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:0.0:1787829872.509545:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829872.509549:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.509550:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509559:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:0.0:1787829872.509560:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829872.509562:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509565:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:0.0:1787829872.509566:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:1.0:1787829872.509567:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509571:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:0.0:1787829872.509574:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787829872.509574:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829872.509579:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:0.0:1787829872.509580:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829872.509583:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829872.509586:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787829872.509586:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:0.0:1787829872.509590:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:1.0:1787829872.509592:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829872.509596:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829872.509604:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530800.
00000100:00100000:1.0:1787829872.509606:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327387520:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829872.509612:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509614:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509618:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1787829872.509622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787829872.509626:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327387520/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.509650:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829872.509651:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829872.509672:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829872.509675:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829872.509677:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829872.509679:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829872.509680:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829872.509683:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829872.509685:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533800.
02000000:00000001:1.0:1787829872.509688:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829872.509689:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829872.509690:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829872.509693:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829872.509694:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509695:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509699:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829872.509705:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000226d20b5] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509706:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829872.509709:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509713:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829872.509715:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829872.509717:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829872.509723:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829872.509727:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829872.509736:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4030)
00000100:00000001:1.0:1787829872.509746:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829872.509748:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.509751:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829872.509754:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829872.509756:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829872.509761:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387648:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829872.509765:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829872.509767:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829872.509769:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.509771:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829872.509774:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829872.509775:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829872.509779:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829872.509781:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829872.509783:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829872.509786:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387648, portal 10
00000100:00000001:1.0:1787829872.509787:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829872.509788:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829872.509793:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509798:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829872.509800:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829872.509801:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829872.509803:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327387648, offset 0
00000400:00000200:1.0:1787829872.509805:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829872.509808:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.509811:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829872.509813:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829872.509815:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829872.509817:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829872.509818:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829872.509821:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40800:40800:256:4294967295] 192.168.203.116@tcp LPNI seq info [40800:40800:8:4294967295]
00000400:00000200:1.0:1787829872.509824:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.509828:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829872.509832:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829872.509834:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829872.509837:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829872.509839:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829872.509842:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829872.509844:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829872.509846:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829872.509847:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829872.509850:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829872.509852:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829872.509854:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829872.509855:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829872.509865:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829872.509868:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509873:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829872.509875:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509880:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509881:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829872.509883:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829872.509886:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829872.509888:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829872.509890:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298692619
00000100:00000001:1.0:1787829872.509892:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298693292
00000100:00000001:1.0:1787829872.509893:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298692619
00000100:00000001:1.0:1787829872.509895:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298693292
00000100:00000001:1.0:1787829872.509896:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829872.509898:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.509899:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829872.509901:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872]  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829872.509909:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829872.509910:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829872.509912:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829872.509915:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829872.509916:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829872.509919:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829872.509922:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829872.509931:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509938:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829872.509940:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509942:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829872.509943:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829872.509947:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509955:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829872.509959:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509963:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829872.509965:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509966:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829872.509967:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.509969:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829872.509970:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829872.509972:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829872.509974:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829872.509976:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829872.509978:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829872.509983:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387648:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829872.509986:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829872.509988:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.509992:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829872.509993:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.509994:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829872.509995:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829872.509997:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829872.509998:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387776:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829872.510001:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829872.510002:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829872.510003:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.510004:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829872.510007:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829872.510008:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829872.510011:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829872.510013:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829872.510015:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829872.510017:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387776, portal 4
00000100:00000001:1.0:1787829872.510019:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829872.510020:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829872.510024:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510028:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829872.510030:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829872.510031:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829872.510033:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327387776, offset 0
00000400:00000200:1.0:1787829872.510035:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829872.510038:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.510041:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829872.510043:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829872.510045:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829872.510047:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829872.510048:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829872.510050:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40801:40801:256:4294967295] 192.168.203.116@tcp LPNI seq info [40801:40801:8:4294967295]
00000400:00000200:1.0:1787829872.510053:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.510057:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829872.510060:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829872.510062:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829872.510065:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829872.510067:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829872.510070:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829872.510071:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829872.510074:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829872.510075:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829872.510077:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829872.510080:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829872.510081:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829872.510083:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829872.510085:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829872.510087:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510092:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829872.510094:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510098:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.510099:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829872.510101:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829872.510103:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829872.510105:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829872.510107:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298692626
00000100:00000001:1.0:1787829872.510108:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298693292
00000100:00000001:1.0:1787829872.510110:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298692626
00000100:00000001:1.0:1787829872.510111:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298693292
00000100:00000001:1.0:1787829872.510113:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829872.510114:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.510115:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829872.510119:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872]  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829872.510126:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829872.510128:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829872.510130:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829872.510132:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829872.510134:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829872.510136:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829872.510140:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829872.510149:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510157:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829872.510159:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510161:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829872.510162:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829872.510166:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510172:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829872.510175:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510180:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829872.510181:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510182:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829872.510183:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510185:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829872.510186:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829872.510187:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829872.510189:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829872.510190:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829872.510193:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829872.510197:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387776:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829872.510200:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510201:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829872.510203:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327387648/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510208:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829872.510209:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829872.510210:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4800.
02000000:00000001:1.0:1787829872.510212:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829872.510213:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829872.510213:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829872.510215:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829872.510216:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533400.
02000000:00000001:1.0:1787829872.510218:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829872.510219:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829872.510220:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829872.510222:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829872.510223:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510224:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829872.510226:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327387776/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510230:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829872.510231:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829872.510232:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829872.510233:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829872.510234:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829872.510235:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829872.510236:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829872.510237:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530000.
02000000:00000001:1.0:1787829872.510238:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829872.510238:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829872.510239:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829872.510241:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829872.510242:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510243:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510245:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829872.510246:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.510247:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829872.510248:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829872.510249:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829872.510251:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510255:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829872.510256:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.510257:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829872.510259:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829872.510260:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829872.510262:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387904:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829872.510264:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829872.510265:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829872.510266:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829872.510268:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829872.510270:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829872.510272:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829872.510274:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829872.510276:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829872.510278:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829872.510280:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327387904, portal 4
00000100:00000001:1.0:1787829872.510281:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829872.510282:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829872.510286:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510291:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829872.510292:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829872.510294:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829872.510295:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327387904, offset 0
00000400:00000200:1.0:1787829872.510298:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829872.510301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.510303:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829872.510305:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829872.510307:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829872.510308:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829872.510310:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829872.510312:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40802:40802:256:4294967295] 192.168.203.116@tcp LPNI seq info [40802:40802:8:4294967295]
00000400:00000200:1.0:1787829872.510315:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829872.510319:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829872.510321:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829872.510323:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829872.510340:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829872.510342:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787829872.510345:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829872.510347:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787829872.510349:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829872.510351:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829872.510353:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829872.510355:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829872.510358:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829872.510359:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829872.510361:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829872.510363:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510368:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829872.510370:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510374:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.510376:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829872.510378:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829872.510380:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829872.510382:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829872.510384:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298692631
00000100:00000001:1.0:1787829872.510385:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298693292
00000100:00000001:1.0:1787829872.510387:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298692631
00000100:00000001:1.0:1787829872.510388:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298693292
00000100:00000001:1.0:1787829872.510389:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829872.510391:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.510392:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829872.510394:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872]  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829872.510399:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829872.510400:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829872.510401:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829872.510402:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829872.510404:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829872.510405:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829872.510407:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829872.510416:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510421:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829872.510421:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510422:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829872.510423:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829872.510425:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510429:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829872.510432:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510436:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829872.510437:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510438:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829872.510439:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510440:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829872.510441:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829872.510443:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829872.510444:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829872.510446:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829872.510448:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829872.510452:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327387904:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829872.510455:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510456:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829872.510458:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327387904/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829872.510466:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829872.510467:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829872.510469:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f4400.
02000000:00000001:1.0:1787829872.510470:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829872.510471:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829872.510472:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829872.510473:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829872.510474:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530800.
02000000:00000001:1.0:1787829872.510475:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829872.510476:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829872.510477:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829872.510479:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829872.510479:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510481:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829872.510483:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829872.510484:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829872.510485:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829872.510486:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829873.020789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829873.020796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829873.020799:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829873.020804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829873.020807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829873.020809:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829873.020811:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829873.020813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829873.020815:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829873.212751:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298693292, at 4298693312
00000100:00000040:2.0:1787829873.212754:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298693292, at 4298693312
00000100:00000010:2.0:1787829873.212766:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ea0.
00000100:00000010:3.0:1787829873.212769:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e635a0.
00010000:00000001:0.0:1787829873.340767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829873.340772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829873.340776:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829873.340777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829873.340779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829873.340781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829873.340848:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829873.340852:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829873.340856:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829874.044760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829874.044766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829874.044768:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829874.044772:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829874.044775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829874.044778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829874.044780:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829874.044782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829874.044784:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829874.364755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829874.364760:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829874.364765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829874.364766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829874.364768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829874.364769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829874.364786:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829874.364788:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829874.364790:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829875.068769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829875.068776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829875.068779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829875.068784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829875.068789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829875.068792:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829875.068795:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829875.068797:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829875.068799:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829875.388762:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829875.388771:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829875.388773:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388777:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829875.388779:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388781:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388784:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829875.388789:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829875.388790:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829875.388792:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388793:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829875.388795:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388796:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388798:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829875.388801:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829875.388802:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829875.388858:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829875.388862:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829875.388866:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829876.092819:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829876.092828:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829876.092830:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829876.092836:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829876.092840:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829876.092844:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829876.092848:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829876.092851:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829876.092854:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829876.412784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829876.412790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829876.412796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829876.412798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829876.412802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829876.412804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829876.412870:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829876.412874:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829876.412878:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829877.116769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829877.116775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829877.116777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829877.116781:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829877.116783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829877.116786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.116788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829877.116790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.116792:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829877.436785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829877.436792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829877.436797:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829877.436799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829877.436802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829877.436803:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829877.436820:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829877.436830:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829877.436833:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829877.628780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829877.628790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829877.628793:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829877.628797:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829877.628801:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829877.628808:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4025
00000400:00000200:1.0:1787829877.628818:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829877.628824:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829877.628828:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829877.628832:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829877.628835:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829877.628838:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829877.628841:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829877.628846:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829877.628849:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829877.628851:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829877.628855:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829877.628857:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829877.628860:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829877.628864:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829877.628866:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829877.628868:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829877.628870:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829877.628874:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829877.628878:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829877.628880:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829877.628883:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829877.628886:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829877.628889:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.628891:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829877.628894:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829877.628904:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829877.628907:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829877.628911:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829877.628913:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.628916:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829877.628918:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829877.628922:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829877.628930:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829877.628936:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829877.628945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829877.628956:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829877.628960:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829877.628965:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829877.628996:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829877.628999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829877.629003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829877.629004:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829877.629006:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829877.629010:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829877.629013:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4025
00000400:00000200:1.0:1787829877.629019:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829877.629022:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829877.629025:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787829877.629026:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829877.629030:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787829877.629032:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829877.629034:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829877.629037:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829877.629043:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829877.629043:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829877.629049:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829877.629055:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829877.629058:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829877.629059:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1787829877.629061:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829877.629062:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787829877.629066:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
02000000:00000001:0.0:1787829877.629067:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829877.629070:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1787829877.629071:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:0.0:1787829877.629077:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829877.629078:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00100000:0.0:1787829877.629083:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327388032:192.168.203.116@tcp:250:kworker.0
00000100:00000040:1.0:1787829877.629087:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829877.629089:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829877.629092:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829877.629094:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829877.629098:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829877.629098:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829877.629103:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829877.629104:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829877.629108:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000010:0.0:1787829877.629109:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829877.629114:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829877.629114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829877.629121:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.629122:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829877.629127:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
02000000:00000001:1.0:1787829877.629127:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1787829877.629134:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
02000000:00000001:1.0:1787829877.629136:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1787829877.629141:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388032, portal 25
02000000:00000001:1.0:1787829877.629142:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629146:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829877.629148:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829877.629149:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000008:00100000:1.0:1787829877.629154:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000040:0.0:1787829877.629156:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829877.629162:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.629167:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629168:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829877.629173:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000010:1.0:1787829877.629175:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000400:00000010:0.0:1787829877.629176:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829877.629183:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327388032, offset 0
00000020:00000040:1.0:1787829877.629183:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829877.629190:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829877.629191:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:1.0:1787829877.629193:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.629197:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829877.629198:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829877.629201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829877.629203:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.629207:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000010:1.0:1787829877.629210:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
00000400:00000200:0.0:1787829877.629212:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829877.629218:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.629219:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829877.629224:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829877.629228:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00100000:1.0:1787829877.629230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829877.629233:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40803:40803:256:4294967295] 192.168.203.116@tcp LPNI seq info [40803:40803:8:4294967295]
00000400:00000200:0.0:1787829877.629243:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829877.629252:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829877.629257:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.629258:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829877.629266:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829877.629266:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1787829877.629273:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829877.629282:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829877.629285:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829877.629286:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829877.629306:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829877.629307:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829877.629311:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00080000:1.0:1787829877.629314:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000100:0.0:1787829877.629319:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829877.629319:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000010:0.0:1787829877.629321:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000100:00080000:1.0:1787829877.629324:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829877.629328:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787829877.629329:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829877.629336:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4025
00000400:00000200:0.0:1787829877.629337:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829877.629344:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000200:1.0:1787829877.629346:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829877.629347:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829877.629353:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:1.0:1787829877.629354:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000200:0.0:1787829877.629359:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829877.629359:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829877.629369:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829877.629374:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829877.629374:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829877.629379:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829877.629380:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829877.629384:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829877.629389:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829877.629392:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000001:0.0:1787829877.629394:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829877.629396:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829877.629397:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:1.0:1787829877.629400:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000001:0.0:1787829877.629403:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829877.629404:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829877.629409:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829877.629410:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829877.629415:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298698412
00000100:00000040:1.0:1787829877.629418:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000010:0.0:1787829877.629421:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd80.
00000100:00000001:1.0:1787829877.629422:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829877.629427:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829877.629430:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829877.629430:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829877.629436:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000001:0.0:1787829877.629438:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298698412
00000100:00080000:1.0:1787829877.629441:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000010:0.0:1787829877.629444:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc60.
00000100:00000001:1.0:1787829877.629446:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629448:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:1.0:1787829877.629450:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829877.629455:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829877.629457:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829877.629459:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629461:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629466:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829877.629466:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829877.629471:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1787829877.629472:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877]  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829877.629478:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.629484:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829877.629486:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829877.629489:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829877.629492:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000100:00000010:1.0:1787829877.629492:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000400:00000200:0.0:1787829877.629497:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000020:00000040:1.0:1787829877.629498:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000010:0.0:1787829877.629500:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
02000000:00000001:1.0:1787829877.629505:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829877.629506:0:2346:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:1.0:1787829877.629510:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829877.629511:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829877.629514:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829877.629516:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829877.629520:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829877.629524:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787829877.629527:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:0.0:1787829877.629534:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829877.629536:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629538:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.629543:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:1.0:1787829877.629545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829877.629546:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829877.629557:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829877.629558:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829877.629562:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829877.629572:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.629574:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829877.629585:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829877.629586:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829877.629592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829877.629600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829877.629604:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829877.629608:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829877.629615:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829877.629620:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829877.629623:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4025
00000100:00000001:0.0:1787829877.629625:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1787829877.629630:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829877.629631:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.629634:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1787829877.629637:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829877.629639:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00080000:1.0:1787829877.629642:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829877.629643:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1787829877.629648:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00080000:0.0:1787829877.629650:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787829877.629652:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:0.0:1787829877.629683:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829877.629686:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829877.629691:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829877.629697:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00100000:0.0:1787829877.629703:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327388032:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829877.629704:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829877.629711:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829877.629712:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:1.0:1787829877.629715:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829877.629716:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:1.0:1787829877.629720:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1787829877.629723:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327388032/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829877.629724:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829877.629729:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829877.629735:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829877.629743:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000040:1.0:1787829877.629744:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829877.629748:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:1.0:1787829877.629749:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:0.0:1787829877.629753:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
00000100:00000040:1.0:1787829877.629755:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829877.629759:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:1.0:1787829877.629761:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829877.629763:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829877.629768:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000040:1.0:1787829877.629768:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000020:00000001:0.0:1787829877.629775:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00080000:1.0:1787829877.629776:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000010:0.0:1787829877.629778:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829877.629784:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.629785:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829877.629787:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829877.629788:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829877.629791:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
02000000:00000001:1.0:1787829877.629792:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829877.629794:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829877.629797:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829877.629798:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629800:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829877.629803:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829877.629806:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829877.629808:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000008:00000001:1.0:1787829877.629809:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629811:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629814:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829877.629814:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629817:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829877.629819:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:1.0:1787829877.629821:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000100:00000040:0.0:1787829877.629825:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829877.629826:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829877.629832:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829877.629834:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.629839:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829877.629842:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829877.629844:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829877.629845:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629848:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.629852:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:1.0:1787829877.629853:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
02000000:00000001:0.0:1787829877.629855:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829877.629860:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829877.629862:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829877.629865:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388160:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829877.629872:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:1.0:1787829877.629874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829877.629876:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829877.629878:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.629881:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829877.629885:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829877.629889:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.629895:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829877.629900:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000001:1.0:1787829877.629901:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1787829877.629903:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829877.629907:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388160, portal 10
00000100:00000001:1.0:1787829877.629908:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.629909:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829877.629911:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829877.629923:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829877.629924:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.629935:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829877.629940:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829877.629943:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829877.629947:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327388160, offset 0
00000400:00000200:0.0:1787829877.629953:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829877.629957:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829877.629957:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.629963:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829877.629966:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:1.0:1787829877.629968:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4035)
00000400:00000200:0.0:1787829877.629969:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829877.629973:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829877.629977:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829877.629980:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40804:40804:256:4294967295] 192.168.203.116@tcp LPNI seq info [40804:40804:8:4294967295]
00000400:00000200:0.0:1787829877.629983:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829877.629988:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829877.629993:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829877.629996:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829877.629999:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829877.630001:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829877.630005:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829877.630006:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829877.630009:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829877.630010:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829877.630013:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829877.630015:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829877.630017:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829877.630019:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829877.630020:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829877.630023:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630029:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829877.630031:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630035:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630036:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829877.630038:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829877.630041:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829877.630043:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829877.630044:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298697739
00000100:00000001:0.0:1787829877.630047:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298698412
00000100:00000001:0.0:1787829877.630048:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298697739
00000100:00000001:0.0:1787829877.630050:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298698412
00000100:00000001:0.0:1787829877.630051:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829877.630052:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630054:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829877.630056:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877]  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829877.630061:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829877.630062:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829877.630063:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829877.630064:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829877.630065:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829877.630067:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829877.630069:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829877.630074:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630078:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829877.630078:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630080:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.630080:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829877.630083:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630087:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829877.630089:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630094:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829877.630095:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630097:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.630098:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630099:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829877.630101:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829877.630103:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829877.630105:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829877.630106:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829877.630108:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829877.630113:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388160:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829877.630115:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829877.630117:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630121:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829877.630122:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.630123:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829877.630125:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829877.630126:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829877.630128:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388288:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829877.630130:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829877.630131:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829877.630132:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.630133:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829877.630136:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829877.630137:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.630139:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829877.630140:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829877.630142:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829877.630144:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388288, portal 4
00000100:00000001:0.0:1787829877.630145:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829877.630147:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829877.630150:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630154:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829877.630156:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829877.630157:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829877.630159:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327388288, offset 0
00000400:00000200:0.0:1787829877.630161:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829877.630163:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829877.630166:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829877.630167:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829877.630169:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829877.630171:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829877.630172:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829877.630174:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40805:40805:256:4294967295] 192.168.203.116@tcp LPNI seq info [40805:40805:8:4294967295]
00000400:00000200:0.0:1787829877.630177:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829877.630181:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829877.630184:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829877.630185:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829877.630188:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829877.630189:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829877.630192:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829877.630194:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829877.630196:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829877.630197:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829877.630199:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829877.630201:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829877.630203:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829877.630205:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829877.630206:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829877.630208:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630213:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829877.630215:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630220:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630221:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829877.630222:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829877.630224:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829877.630226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829877.630227:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298697746
00000100:00000001:0.0:1787829877.630229:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298698412
00000100:00000001:0.0:1787829877.630230:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298697746
00000100:00000001:0.0:1787829877.630232:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298698412
00000100:00000001:0.0:1787829877.630233:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829877.630234:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630235:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829877.630238:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877]  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829877.630242:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829877.630243:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829877.630244:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829877.630245:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829877.630246:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829877.630247:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829877.630249:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829877.630254:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630258:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829877.630259:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630260:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.630261:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829877.630263:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630268:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829877.630270:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630274:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829877.630276:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630277:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.630277:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630278:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829877.630279:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829877.630281:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829877.630283:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829877.630284:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829877.630286:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829877.630309:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388288:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829877.630312:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630313:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829877.630315:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327388160/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630319:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829877.630320:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829877.630321:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829877.630323:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829877.630324:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829877.630325:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829877.630327:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829877.630328:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4800.
02000000:00000001:0.0:1787829877.630330:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829877.630331:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829877.630332:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829877.630334:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829877.630335:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630336:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829877.630338:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327388288/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630342:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829877.630343:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829877.630343:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829877.630345:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829877.630345:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829877.630346:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829877.630347:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829877.630348:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:0.0:1787829877.630349:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829877.630350:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829877.630351:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829877.630352:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829877.630353:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630354:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630356:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829877.630357:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630358:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829877.630359:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829877.630360:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829877.630362:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630367:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829877.630368:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.630369:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829877.630371:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829877.630372:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829877.630374:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388416:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829877.630376:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829877.630378:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829877.630379:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829877.630380:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829877.630382:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829877.630383:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829877.630386:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829877.630387:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829877.630389:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829877.630391:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388416, portal 4
00000100:00000001:0.0:1787829877.630392:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829877.630393:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:0.0:1787829877.630397:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630401:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829877.630403:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829877.630404:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829877.630405:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327388416, offset 0
00000400:00000200:0.0:1787829877.630408:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829877.630411:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829877.630413:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829877.630415:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829877.630416:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829877.630418:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829877.630420:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829877.630422:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40806:40806:256:4294967295] 192.168.203.116@tcp LPNI seq info [40806:40806:8:4294967295]
00000400:00000200:0.0:1787829877.630424:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829877.630429:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829877.630432:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829877.630434:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829877.630436:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829877.630439:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829877.630441:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829877.630443:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829877.630446:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829877.630447:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829877.630449:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829877.630451:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829877.630453:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829877.630454:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829877.630456:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829877.630458:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630463:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829877.630465:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630469:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630470:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829877.630472:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829877.630474:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829877.630476:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829877.630477:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298697751
00000100:00000001:0.0:1787829877.630479:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298698412
00000100:00000001:0.0:1787829877.630480:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298697751
00000100:00000001:0.0:1787829877.630482:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298698412
00000100:00000001:0.0:1787829877.630483:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829877.630484:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630485:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829877.630487:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877]  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829877.630492:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829877.630493:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829877.630494:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829877.630495:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829877.630497:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829877.630498:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829877.630500:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829877.630505:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630509:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829877.630510:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630511:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.630512:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829877.630514:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630518:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829877.630520:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630524:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829877.630526:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630527:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829877.630528:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630529:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829877.630530:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829877.630532:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829877.630533:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829877.630535:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829877.630537:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829877.630541:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388416:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829877.630544:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630546:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829877.630548:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327388416/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829877.630552:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829877.630552:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829877.630554:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829877.630555:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829877.630555:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829877.630556:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829877.630557:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829877.630558:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:0.0:1787829877.630559:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829877.630560:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829877.630562:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829877.630563:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829877.630564:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630565:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829877.630567:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829877.630568:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829877.630570:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829877.630571:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829878.140757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829878.140762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829878.140764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829878.140768:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829878.140771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829878.140774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829878.140776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829878.140778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829878.140780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829878.332768:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298698412, at 4298698432
00000100:00000040:3.0:1787829878.332768:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298698412, at 4298698432
00000100:00000010:2.0:1787829878.332784:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd80.
00000100:00000010:3.0:1787829878.332787:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc60.
00010000:00000001:0.0:1787829878.460761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829878.460767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829878.460772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829878.460773:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829878.460776:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829878.460777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829878.460832:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829878.460834:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829878.460837:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829879.164747:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829879.164753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829879.164754:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829879.164758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829879.164761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829879.164764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829879.164766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829879.164768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829879.164769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829879.484764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829879.484772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829879.484778:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829879.484779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829879.484782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829879.484784:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829879.484844:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829879.484847:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829879.484849:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829880.188767:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829880.188773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829880.188774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829880.188778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829880.188780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829880.188783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829880.188785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829880.188787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829880.188789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829880.508750:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829880.508756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829880.508761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829880.508762:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829880.508764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829880.508767:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829880.508768:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829880.508771:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829880.508773:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829880.508775:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829880.508777:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829880.508778:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829880.508794:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829880.508796:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829880.508799:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829881.212764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829881.212771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829881.212773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829881.212777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829881.212779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829881.212782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829881.212784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829881.212786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829881.212788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829881.532751:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829881.532758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829881.532764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829881.532765:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829881.532768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829881.532769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829881.532822:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829881.532826:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829881.532829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829882.236778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829882.236784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829882.236785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829882.236800:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829882.236808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829882.236813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.236816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829882.236819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.236822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829882.556757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829882.556765:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829882.556771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829882.556772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829882.556775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829882.556776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829882.556836:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829882.556840:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829882.556844:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829882.748739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829882.748749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829882.748752:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829882.748756:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829882.748761:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829882.748768:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4030
00000400:00000200:1.0:1787829882.748775:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829882.748781:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829882.748783:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829882.748786:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.748788:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829882.748790:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.748792:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829882.748795:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829882.748796:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829882.748798:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829882.748800:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829882.748802:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829882.748804:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829882.748806:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.748807:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829882.748809:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.748810:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829882.748812:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829882.748814:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829882.748816:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829882.748823:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829882.748825:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829882.748827:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.748829:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829882.748831:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829882.748838:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829882.748842:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829882.748846:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829882.748848:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.748851:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829882.748854:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829882.748858:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829882.748865:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:1.0:1787829882.748872:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829882.748881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829882.748892:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829882.748895:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829882.748899:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829882.748916:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829882.748919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829882.748921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829882.748922:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829882.748924:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829882.748927:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829882.748931:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4030
00000400:00000200:1.0:1787829882.748936:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829882.748939:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829882.748942:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787829882.748943:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829882.748947:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787829882.748951:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829882.748952:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787829882.748957:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829882.748960:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829882.748963:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.748969:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829882.748975:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1787829882.748976:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829882.748978:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.748981:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829882.748983:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829882.748986:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
02000000:00000001:0.0:1787829882.748987:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829882.748989:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:0.0:1787829882.748993:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327388544:192.168.203.116@tcp:250:kworker.0
00000020:00000040:1.0:1787829882.748994:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829882.749000:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:1.0:1787829882.749000:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829882.749004:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
02000000:00000001:0.0:1787829882.749005:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829882.749009:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829882.749009:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749014:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829882.749015:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1787829882.749019:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749021:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000010:0.0:1787829882.749022:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
00000020:00000040:1.0:1787829882.749023:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829882.749026:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829882.749028:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749032:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.749035:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829882.749036:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1787829882.749041:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
02000000:00000001:1.0:1787829882.749043:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829882.749046:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829882.749048:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829882.749048:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829882.749052:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829882.749055:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829882.749056:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388544, portal 25
00000100:00000001:1.0:1787829882.749056:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749061:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829882.749064:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000010:1.0:1787829882.749064:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829882.749070:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000040:0.0:1787829882.749074:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829882.749075:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829882.749078:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.749082:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829882.749084:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829882.749086:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749088:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:1.0:1787829882.749090:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000100:00000040:0.0:1787829882.749096:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749096:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829882.749101:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00100000:1.0:1787829882.749104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1787829882.749107:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327388544, offset 0
00000100:00000001:1.0:1787829882.749115:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829882.749117:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.749121:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829882.749122:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829882.749134:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829882.749142:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.749145:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00080000:1.0:1787829882.749146:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829882.749152:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829882.749153:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829882.749154:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829882.749157:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829882.749160:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:1.0:1787829882.749161:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829882.749164:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1787829882.749167:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4030
00000400:00000200:0.0:1787829882.749170:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829882.749174:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40807:40807:256:4294967295] 192.168.203.116@tcp LPNI seq info [40807:40807:8:4294967295]
00000400:00000200:0.0:1787829882.749177:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829882.749182:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829882.749189:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829882.749192:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1787829882.749193:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829882.749200:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749205:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1787829882.749206:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829882.749210:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:0.0:1787829882.749212:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:1.0:1787829882.749212:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829882.749217:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829882.749218:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829882.749219:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829882.749222:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:0.0:1787829882.749223:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000001:1.0:1787829882.749227:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829882.749228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1787829882.749230:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829882.749237:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749239:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829882.749241:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749242:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829882.749243:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829882.749245:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.1:1787829882.749246:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829882.749247:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829882.749252:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787829882.749252:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829882.749271:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:1.0:1787829882.749271:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:0.0:1787829882.749273:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
02000000:00000001:1.0:1787829882.749277:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.749281:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829882.749282:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:1.0:1787829882.749284:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829882.749289:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000008:00000001:1.0:1787829882.749290:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.749293:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.749295:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000100:00000010:1.0:1787829882.749297:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000400:00000010:0.0:1787829882.749298:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000020:00000040:1.0:1787829882.749301:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:0.0:1787829882.749304:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:1.0:1787829882.749305:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829882.749307:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829882.749310:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829882.749310:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829882.749311:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829882.749314:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829882.749320:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:0.0:1787829882.749325:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829882.749325:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829882.749330:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829882.749334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829882.749348:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749351:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829882.749351:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829882.749355:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749358:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829882.749363:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829882.749364:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829882.749368:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298703532
00000100:00000010:0.0:1787829882.749374:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc3c0.
00000100:00000001:1.0:1787829882.749380:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749383:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:1.0:1787829882.749385:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829882.749388:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298703532
00000100:00080000:1.0:1787829882.749391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:0.0:1787829882.749393:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccc00.
00000100:00000001:1.0:1787829882.749394:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829882.749397:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:1.0:1787829882.749398:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829882.749400:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829882.749404:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.749405:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829882.749407:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:1.0:1787829882.749410:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4030
00000100:00100000:0.0:1787829882.749414:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882]  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829882.749419:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829882.749425:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829882.749429:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829882.749435:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749439:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829882.749443:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749446:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000001:0.0:1787829882.749447:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829882.749450:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000001:1.0:1787829882.749450:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:0.0:1787829882.749454:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000020:00000040:1.0:1787829882.749454:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829882.749458:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829882.749459:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000020:00000040:1.0:1787829882.749461:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000400:00000010:0.0:1787829882.749462:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000020:00000001:1.0:1787829882.749464:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829882.749466:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:1.0:1787829882.749468:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829882.749473:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829882.749476:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749482:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829882.749487:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749493:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829882.749497:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829882.749498:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829882.749507:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829882.749514:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749515:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:1.0:1787829882.749516:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829882.749518:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829882.749520:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829882.749521:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.749525:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829882.749525:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.749530:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:0.0:1787829882.749534:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1787829882.749534:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829882.749539:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829882.749543:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749550:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:1.0:1787829882.749550:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829882.749555:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000040:0.0:1787829882.749556:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829882.749561:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829882.749564:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749567:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829882.749568:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829882.749572:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829882.749573:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829882.749576:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749577:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.749581:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:1.0:1787829882.749582:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:0.0:1787829882.749584:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829882.749589:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829882.749590:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829882.749595:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00100000:1.0:1787829882.749601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:0.0:1787829882.749605:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829882.749611:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:1.0:1787829882.749618:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1787829882.749622:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829882.749622:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829882.749629:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829882.749638:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327388544:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829882.749644:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749646:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829882.749648:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4040)
00000100:00000001:0.0:1787829882.749649:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.749654:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327388544/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749682:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829882.749684:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829882.749687:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829882.749690:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829882.749691:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829882.749693:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829882.749695:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829882.749697:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829882.749701:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829882.749702:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829882.749704:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829882.749706:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829882.749707:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.749709:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.749714:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829882.749716:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749717:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829882.749719:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829882.749720:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829882.749722:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749727:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829882.749728:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.749730:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829882.749731:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829882.749733:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829882.749735:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388672:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829882.749737:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829882.749739:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829882.749740:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.749741:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829882.749744:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829882.749745:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.749748:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829882.749750:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829882.749752:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829882.749755:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388672, portal 10
00000100:00000001:0.0:1787829882.749756:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829882.749757:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829882.749762:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749766:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829882.749768:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829882.749769:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829882.749771:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327388672, offset 0
00000400:00000200:0.0:1787829882.749774:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829882.749776:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829882.749779:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829882.749782:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829882.749785:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829882.749798:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829882.749800:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829882.749804:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40808:40808:256:4294967295] 192.168.203.116@tcp LPNI seq info [40808:40808:8:4294967295]
00000400:00000200:0.0:1787829882.749809:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829882.749816:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829882.749821:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829882.749826:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829882.749830:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829882.749834:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829882.749838:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829882.749841:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829882.749844:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829882.749845:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829882.749847:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829882.749850:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829882.749853:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829882.749854:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829882.749856:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829882.749858:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749864:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.749866:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749870:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749871:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829882.749874:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829882.749877:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829882.749879:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829882.749881:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298702859
00000100:00000001:0.0:1787829882.749883:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298703532
00000100:00000001:0.0:1787829882.749884:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298702859
00000100:00000001:0.0:1787829882.749886:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298703532
00000100:00000001:0.0:1787829882.749887:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829882.749888:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.749890:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829882.749892:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882]  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829882.749897:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829882.749899:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829882.749900:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829882.749902:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829882.749903:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829882.749904:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829882.749906:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829882.749911:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749915:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829882.749916:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.749917:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.749918:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829882.749920:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749924:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829882.749927:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749931:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829882.749933:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.749934:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.749935:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.749936:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829882.749937:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829882.749939:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829882.749941:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829882.749942:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829882.749944:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829882.749949:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388672:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829882.749952:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829882.749955:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749959:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829882.749960:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.749962:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829882.749963:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829882.749964:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829882.749966:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388800:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829882.749968:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829882.749969:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829882.749971:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.749972:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829882.749974:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829882.749976:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.749978:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829882.749980:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829882.749982:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829882.749984:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388800, portal 4
00000100:00000001:0.0:1787829882.749985:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829882.749986:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829882.749990:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.749994:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829882.749996:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829882.749997:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829882.749999:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327388800, offset 0
00000400:00000200:0.0:1787829882.750001:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829882.750003:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829882.750006:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829882.750008:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829882.750010:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829882.750013:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829882.750015:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829882.750018:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40809:40809:256:4294967295] 192.168.203.116@tcp LPNI seq info [40809:40809:8:4294967295]
00000400:00000200:0.0:1787829882.750022:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829882.750029:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829882.750034:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829882.750038:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829882.750042:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829882.750045:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829882.750050:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829882.750052:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829882.750056:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829882.750058:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829882.750060:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829882.750062:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829882.750065:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829882.750066:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829882.750068:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829882.750070:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750076:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.750078:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750083:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.750084:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829882.750086:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829882.750088:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829882.750091:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829882.750093:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298702866
00000100:00000001:0.0:1787829882.750094:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298703532
00000100:00000001:0.0:1787829882.750096:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298702866
00000100:00000001:0.0:1787829882.750098:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298703532
00000100:00000001:0.0:1787829882.750099:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829882.750101:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.750102:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829882.750104:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882]  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829882.750109:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829882.750110:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829882.750111:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829882.750113:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829882.750114:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829882.750115:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829882.750117:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829882.750122:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750126:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829882.750127:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750128:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.750129:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829882.750131:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750135:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829882.750137:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750142:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829882.750144:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750145:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.750146:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750147:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829882.750148:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829882.750150:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829882.750153:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829882.750154:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829882.750156:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829882.750161:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388800:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829882.750163:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829882.750165:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750169:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829882.750170:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.750172:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829882.750173:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829882.750174:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829882.750176:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388928:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829882.750178:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829882.750179:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829882.750180:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829882.750181:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829882.750184:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829882.750185:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829882.750187:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829882.750189:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829882.750191:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829882.750193:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327388928, portal 4
00000100:00000001:0.0:1787829882.750194:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829882.750195:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:0.0:1787829882.750199:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750203:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829882.750205:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829882.750206:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000200:0.0:1787829882.750208:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327388928, offset 0
00000400:00000200:0.0:1787829882.750210:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829882.750212:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829882.750214:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829882.750216:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829882.750218:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829882.750219:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829882.750221:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829882.750223:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40810:40810:256:4294967295] 192.168.203.116@tcp LPNI seq info [40810:40810:8:4294967295]
00000400:00000200:0.0:1787829882.750226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829882.750230:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829882.750233:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829882.750235:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000800:00000200:0.0:1787829882.750237:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829882.750239:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787829882.750242:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829882.750244:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787829882.750246:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829882.750247:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:0.0:1787829882.750249:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829882.750251:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829882.750253:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:0.0:1787829882.750271:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:0.0:1787829882.750274:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829882.750277:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750282:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.750284:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750293:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.750294:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829882.750296:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829882.750298:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829882.750301:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829882.750302:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298702871
00000100:00000001:0.0:1787829882.750304:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298703532
00000100:00000001:0.0:1787829882.750305:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298702871
00000100:00000001:0.0:1787829882.750307:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298703532
00000100:00000001:0.0:1787829882.750308:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829882.750309:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.750311:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829882.750313:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882]  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829882.750318:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829882.750319:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829882.750320:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829882.750321:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829882.750322:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:0.0:1787829882.750324:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829882.750326:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829882.750331:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750335:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829882.750336:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750337:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.750338:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829882.750340:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750345:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829882.750347:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750351:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829882.750353:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750354:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829882.750355:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750356:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829882.750357:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829882.750359:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829882.750360:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829882.750362:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829882.750364:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829882.750369:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327388928:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829882.750371:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750372:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.750374:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327388672/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750378:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829882.750380:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829882.750381:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829882.750383:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829882.750384:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829882.750385:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829882.750386:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829882.750387:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829882.750389:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829882.750390:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829882.750391:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829882.750393:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829882.750394:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750395:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.750397:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327388800/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750401:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829882.750402:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829882.750403:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829882.750405:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829882.750405:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829882.750406:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829882.750407:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829882.750408:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829882.750409:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829882.750410:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829882.750411:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829882.750413:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829882.750413:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750414:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829882.750416:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327388928/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829882.750420:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829882.750420:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829882.750421:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829882.750422:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829882.750423:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829882.750424:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829882.750425:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829882.750425:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829882.750426:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829882.750427:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829882.750428:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829882.750429:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829882.750431:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750432:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829882.750434:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829882.750435:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829882.750436:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829882.750438:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829883.260745:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829883.260751:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829883.260753:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829883.260758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829883.260761:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829883.260764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829883.260766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829883.260768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829883.260769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829883.452736:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298703532, at 4298703552
00000100:00000040:3.0:1787829883.452737:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298703532, at 4298703552
00000100:00000010:3.0:1787829883.452749:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccc00.
00000100:00000010:2.0:1787829883.452752:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc3c0.
00010000:00000001:0.0:1787829883.580774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829883.580792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829883.580798:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829883.580800:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829883.580803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829883.580804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829883.580825:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829883.580828:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829883.580831:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829884.284809:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829884.284817:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829884.284820:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829884.284825:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829884.284829:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829884.284833:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829884.284835:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829884.284837:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829884.284840:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829884.604774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829884.604781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829884.604787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829884.604789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829884.604792:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829884.604794:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829884.604855:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829884.604860:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829884.604864:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829885.308783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829885.308790:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829885.308792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829885.308798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829885.308802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829885.308806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829885.308808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829885.308810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829885.308813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628795:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829885.628804:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829885.628806:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829885.628810:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628815:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829885.628818:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628820:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628823:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628827:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829885.628829:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829885.628831:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829885.628832:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628834:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829885.628836:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628838:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628840:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829885.628843:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829885.628845:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829885.628911:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829885.628915:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829885.628919:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829886.332788:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829886.332795:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829886.332797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829886.332804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829886.332807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829886.332811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829886.332814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829886.332817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829886.332820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829886.652747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829886.652753:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829886.652757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829886.652758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829886.652760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829886.652761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829886.652778:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829886.652780:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829886.652782:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829887.356738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829887.356744:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829887.356746:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829887.356752:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829887.356756:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829887.356760:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829887.356762:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829887.356765:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829887.356768:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829887.676779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829887.676786:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829887.676791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829887.676792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829887.676795:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829887.676796:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829887.676871:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829887.676876:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829887.676880:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829887.868788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829887.868799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829887.868802:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829887.868806:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829887.868811:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829887.868819:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4035
00000400:00000200:2.0:1787829887.868829:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829887.868835:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829887.868838:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829887.868842:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.868846:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829887.868849:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.868852:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829887.868856:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829887.868858:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829887.868861:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829887.868865:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829887.868867:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829887.868870:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829887.868874:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.868876:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829887.868878:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.868881:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829887.868885:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829887.868889:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829887.868892:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829887.868895:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829887.868899:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829887.868902:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.868904:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829887.868907:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829887.868917:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829887.868922:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829887.868926:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829887.868927:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.868931:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829887.868934:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829887.868938:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829887.868946:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829887.868952:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829887.868963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829887.868975:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829887.868980:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829887.868985:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829887.869010:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829887.869014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829887.869017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829887.869019:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829887.869021:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829887.869024:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829887.869027:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4035
00000400:00000200:2.0:1787829887.869032:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829887.869037:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829887.869039:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829887.869044:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869047:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829887.869050:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869052:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829887.869054:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787829887.869056:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829887.869059:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829887.869060:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787829887.869062:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829887.869065:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1787829887.869066:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829887.869071:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829887.869075:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829887.869076:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829887.869084:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869090:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829887.869095:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787829887.869096:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829887.869100:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869101:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829887.869106:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787829887.869107:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
02000000:00000001:0.0:1787829887.869111:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1787829887.869114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:0.0:1787829887.869117:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869120:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829887.869126:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327389056:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787829887.869126:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829887.869134:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787829887.869135:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829887.869140:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787829887.869141:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869146:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829887.869151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829887.869154:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787829887.869155:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869159:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.869162:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:2.0:1787829887.869165:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000020:00000040:2.0:1787829887.869170:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829887.869173:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000010:0.0:1787829887.869174:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:2.0:1787829887.869175:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.869179:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829887.869186:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829887.869193:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000100:00000001:2.0:1787829887.869195:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829887.869200:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1787829887.869201:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
02000000:00000001:2.0:1787829887.869203:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829887.869210:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389056, portal 25
02000000:00000010:2.0:1787829887.869211:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:0.0:1787829887.869216:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829887.869231:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00000001:2.0:1787829887.869232:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829887.869244:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829887.869246:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829887.869268:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829887.869270:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829887.869278:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.869280:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829887.869288:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829887.869294:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef088.
00000100:00000001:2.0:1787829887.869298:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829887.869300:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327389056, offset 0
00000100:00080000:2.0:1787829887.869303:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829887.869309:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829887.869311:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869312:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829887.869317:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829887.869321:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829887.869324:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829887.869327:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4035
00000400:00000200:0.0:1787829887.869336:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829887.869337:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829887.869346:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829887.869348:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829887.869353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829887.869354:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000040:2.0:1787829887.869360:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829887.869362:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787829887.869367:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829887.869369:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:2.0:1787829887.869373:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829887.869380:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40811:40811:256:4294967295] 192.168.203.116@tcp LPNI seq info [40811:40811:8:4294967295]
00000100:00000001:2.0:1787829887.869380:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829887.869387:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829887.869390:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:0.0:1787829887.869391:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:2.0:1787829887.869394:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829887.869398:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829887.869401:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829887.869407:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829887.869411:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1787829887.869416:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869421:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1787829887.869423:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787829887.869428:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:0.0:1787829887.869432:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7300.
00000100:00000001:2.0:1787829887.869433:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829887.869439:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.0:1787829887.869445:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787829887.869446:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869453:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829887.869457:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787829887.869458:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829887.869463:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829887.869467:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869469:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829887.869472:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829887.869475:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869476:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.869480:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829887.869482:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000400:00000200:0.0:1787829887.869488:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787829887.869488:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829887.869494:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829887.869495:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298708652
02000000:00000001:2.0:1787829887.869499:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869503:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:0.0:1787829887.869504:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc960.
02000000:00000001:2.0:1787829887.869505:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829887.869510:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.869515:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000010:2.0:1787829887.869517:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000100:00000001:0.0:1787829887.869522:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298708652
00000100:00000001:2.0:1787829887.869524:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829887.869529:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc240.
00000100:00100000:2.0:1787829887.869534:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829887.869536:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787829887.869547:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829887.869550:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829887.869551:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.869558:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829887.869562:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.869565:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.869571:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829887.869580:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.869582:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1787829887.869584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829887.869587:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00080000:2.0:1787829887.869593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829887.869595:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829887.869597:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829887.869598:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829887.869602:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787829887.869602:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829887.869608:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:0.0:1787829887.869611:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829887.869615:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4035
00000400:00000200:2.0:1787829887.869626:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829887.869629:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829887.869634:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829887.869634:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829887.869640:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
02000000:00000001:0.0:1787829887.869641:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829887.869644:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787829887.869648:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829887.869650:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829887.869684:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389184:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829887.869689:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829887.869692:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787829887.869693:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829887.869694:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.869698:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787829887.869701:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000010:0.0:1787829887.869704:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000100:00000001:2.0:1787829887.869707:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
02000000:00000001:0.0:1787829887.869709:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829887.869713:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829887.869717:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787829887.869717:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000010:0.0:1787829887.869722:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000020:00000001:2.0:1787829887.869723:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829887.869727:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000400:00000010:0.0:1787829887.869729:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000020:00000001:2.0:1787829887.869731:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829887.869735:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389184, portal 10
00000020:00000001:2.0:1787829887.869737:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829887.869739:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829887.869743:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:2.0:1787829887.869745:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869750:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829887.869755:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829887.869757:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869763:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829887.869769:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829887.869772:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787829887.869776:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000040:0.0:1787829887.869779:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829887.869784:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829887.869785:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef330.
02000000:00000001:2.0:1787829887.869789:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1787829887.869792:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327389184, offset 0
02000000:00000001:2.0:1787829887.869795:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829887.869801:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829887.869802:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869807:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829887.869809:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:2.0:1787829887.869811:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829887.869815:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829887.869819:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829887.869824:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000008:00000001:2.0:1787829887.869824:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869829:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829887.869831:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829887.869835:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000010:2.0:1787829887.869837:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000400:00000200:0.0:1787829887.869842:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40812:40812:255:4294967295] 192.168.203.116@tcp LPNI seq info [40812:40812:7:4294967295]
00000020:00000040:2.0:1787829887.869844:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829887.869850:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1787829887.869853:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829887.869854:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829887.869859:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829887.869862:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829887.869866:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829887.869867:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000010:2.0:1787829887.869873:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829887.869878:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829887.869886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829887.869889:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829887.869896:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000100:00000001:2.0:1787829887.869899:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1787829887.869904:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829887.869906:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829887.869915:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829887.869917:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.869921:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829887.869928:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000100:00000001:2.0:1787829887.869933:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829887.869935:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829887.869940:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4045)
00000100:00000001:0.0:1787829887.869947:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298707979
00000100:00000001:0.0:1787829887.869951:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298708652
00000100:00000001:0.0:1787829887.869954:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298707979
00000100:00000001:0.0:1787829887.869957:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298708652
00000100:00000001:0.0:1787829887.869960:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829887.869962:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.869965:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829887.869971:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.869979:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829887.869980:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.869983:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829887.869985:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829887.869987:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829887.869990:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389312:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829887.869994:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829887.869996:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829887.869997:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.870000:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829887.870004:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829887.870006:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829887.870010:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829887.870012:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829887.870016:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829887.870019:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389312, portal 4
00000100:00000001:0.0:1787829887.870021:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829887.870023:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:0.0:1787829887.870030:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.870037:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829887.870040:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829887.870043:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef220.
00000100:00000200:0.0:1787829887.870045:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327389312, offset 0
00000400:00000200:0.0:1787829887.870052:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829887.870056:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829887.870060:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829887.870063:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829887.870065:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829887.870068:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829887.870070:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829887.870074:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40813:40813:254:4294967295] 192.168.203.116@tcp LPNI seq info [40813:40813:6:4294967295]
00000400:00000200:0.0:1787829887.870078:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829887.870085:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829887.870090:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829887.870094:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7000.
00000800:00000200:0.0:1787829887.870098:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829887.870102:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829887.870105:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829887.870108:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829887.870112:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829887.870114:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298707986
00000100:00000001:0.0:1787829887.870116:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298708652
00000100:00000001:0.0:1787829887.870118:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298707986
00000100:00000001:0.0:1787829887.870120:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298708652
00000100:00000001:0.0:1787829887.870123:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829887.870125:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.870127:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.870129:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829887.870132:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829887.870134:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829887.870136:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829887.870137:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829887.870139:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829887.870143:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.870151:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829887.870152:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.870154:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829887.870156:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829887.870158:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829887.870161:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389440:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829887.870167:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829887.870169:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829887.870170:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829887.870172:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829887.870175:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829887.870178:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829887.870181:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829887.870184:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708d68.
00000400:00000010:0.0:1787829887.870187:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef3b8.
00000100:00000200:0.0:1787829887.870190:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389440, portal 4
00000100:00000001:0.0:1787829887.870192:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829887.870194:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:0.0:1787829887.870200:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829887.870206:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829887.870208:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829887.870211:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef5d8.
00000100:00000200:0.0:1787829887.870213:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327389440, offset 0
00000400:00000200:0.0:1787829887.870217:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829887.870238:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829887.870241:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829887.870245:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829887.870247:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829887.870250:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829887.870252:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829887.870255:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40814:40814:253:4294967295] 192.168.203.116@tcp LPNI seq info [40814:40814:5:4294967295]
00000400:00000200:0.0:1787829887.870259:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829887.870265:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829887.870269:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829887.870273:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7f00.
00000800:00000200:0.0:1787829887.870277:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829887.870281:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829887.870284:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829887.870287:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829887.870290:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829887.870292:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298707991
00000100:00000001:0.0:1787829887.870294:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298708652
00000100:00000001:0.0:1787829887.870297:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298707991
00000100:00000001:0.0:1787829887.870299:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298708652
00000100:00000001:0.0:1787829887.870301:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829887.870303:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.870305:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.870307:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829887.870310:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829887.870311:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829887.870313:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829887.870314:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829887.870316:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829887.870318:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829888.380772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829888.380779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829888.380782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829888.380788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829888.380792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829888.380794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829888.380796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829888.380798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829888.380802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829888.572755:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298708652, at 4298708672
00000100:00000040:2.0:1787829888.572756:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298708652, at 4298708672
00000100:00000010:3.0:1787829888.572770:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc240.
00000100:00000010:2.0:1787829888.572773:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc960.
00010000:00000001:0.0:1787829888.700763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829888.700770:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829888.700776:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829888.700778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829888.700782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829888.700783:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829888.700846:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829888.700851:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829888.700854:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829889.404731:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829889.404738:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829889.404740:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829889.404747:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829889.404751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829889.404754:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829889.404757:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829889.404760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829889.404763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829889.724769:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829889.724777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829889.724783:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829889.724784:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829889.724787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829889.724787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829889.724804:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829889.724811:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829889.724814:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829890.428785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829890.428794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829890.428797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829890.428803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829890.428808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829890.428812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.428815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829890.428819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.428822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829890.748783:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829890.748790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829890.748797:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829890.748799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829890.748802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829890.748806:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829890.748817:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829890.748821:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829890.748824:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829890.748826:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829890.748828:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829890.748831:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829890.748886:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829890.748889:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829890.748892:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1787829890.940952:0:2334:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:2.0:1787829890.940988:0:2334:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787829890.940995:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829890.940999:0:2334:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:2.0:1787829890.941003:0:2334:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829890.941006:0:2334:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787829890.941010:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829890.941014:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7300.
00000400:00000200:2.0:1787829890.941017:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829890.941021:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829890.941027:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef088
00000400:00000010:2.0:1787829890.941030:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef088.
00000100:00000001:2.0:1787829890.941035:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829890.941044:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829890.941072:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787829890.941077:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829890.941085:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829890.941087:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829890.941097:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829890.941101:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:2.0:1787829890.941105:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829890.941109:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829890.941113:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef330
00000400:00000010:2.0:1787829890.941115:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef330.
00000100:00000001:2.0:1787829890.941118:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829890.941121:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000200:2.0:1787829890.941125:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941127:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829890.941133:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829890.941139:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1787829890.941146:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829890]  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829890.941147:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829890.941168:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829890.941172:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000001:0.0:1787829890.941179:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829890.941182:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829890.941186:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829890.941191:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:0.0:1787829890.941193:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000800:00000100:2.0:1787829890.941208:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000100:00000001:0.0:1787829890.941211:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000800:00000010:2.0:1787829890.941213:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000200:2.0:1787829890.941218:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000200:0.0:1787829890.941221:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829890.941226:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787829890.941232:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef220
00000400:00000010:2.0:1787829890.941236:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef220.
00000100:00000200:0.0:1787829890.941242:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829890.941242:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787829890.941253:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941268:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829890.941272:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829890.941273:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787829890.941279:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1787829890.941280:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941284:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829890.941293:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829890.941295:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829890.941296:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787829890.941306:0:2334:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:2.0:1787829890.941311:0:2334:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000100:00000001:0.0:1787829890.941315:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1787829890.941316:0:2334:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787829890.941323:0:2334:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000100:00000040:0.0:1787829890.941327:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787829890.941328:0:2334:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef5d8
00000400:00000010:2.0:1787829890.941333:0:2334:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef5d8.
00000100:00000001:2.0:1787829890.941338:0:2334:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787829890.941344:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000200:2.0:1787829890.941347:0:2334:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941349:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941354:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941357:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941362:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829890.941367:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829890.941367:0:2334:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1787829890.941374:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:2.0:1787829890.941374:0:2334:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829890.941385:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829890.941387:0:2334:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829890.941390:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1787829890.941391:0:2334:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787829890.941400:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829890.941409:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327389056:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829890.941414:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829890.941416:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829890.941419:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327389056/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941431:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829890.941434:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829890.941438:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829890.941443:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829890.941446:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829890.941448:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829890.941450:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829890.941454:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:0.0:1787829890.941457:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829890.941459:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829890.941461:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:0.0:1787829890.941465:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829890.941466:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941469:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829890.941471:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829890.941472:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829890.941474:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829890.941478:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829890]  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829890.941487:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829890.941488:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829890.941490:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708d68.
00000400:00000200:0.0:1787829890.941493:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef3b8
00000400:00000010:0.0:1787829890.941495:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef3b8.
00000100:00000001:0.0:1787829890.941498:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829890.941501:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829890.941510:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941517:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829890.941519:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941521:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941522:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829890.941526:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941533:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829890.941538:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941544:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829890.941546:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941548:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941549:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941551:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829890.941553:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829890.941556:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829890.941560:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829890.941563:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829890.941567:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829890.941575:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389440:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829890.941580:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829890.941584:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829890]  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829890.941592:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829890.941594:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829890.941596:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829890.941598:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829890.941600:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829890.941603:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829890.941606:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829890.941614:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941621:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829890.941623:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941625:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941626:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829890.941630:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941637:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829890.941641:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941649:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829890.941651:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941654:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941679:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941681:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829890.941684:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829890.941687:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829890.941692:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829890.941696:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829890.941701:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829890.941710:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389184:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829890.941715:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829890.941719:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829890]  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829890.941728:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829890.941730:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829890.941733:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829890.941736:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829890.941738:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829890.941740:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829890.941743:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829890.941751:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941757:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829890.941759:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941760:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941761:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829890.941765:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941771:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829890.941774:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941780:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829890.941783:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941784:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829890.941785:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941787:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829890.941788:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829890.941791:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829890.941793:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829890.941795:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829890.941799:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829890.941806:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389312:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829890.941810:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941813:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829890.941816:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327389440/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941821:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829890.941823:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829890.941825:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829890.941827:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829890.941829:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829890.941830:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829890.941832:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829890.941834:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829890.941836:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829890.941838:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829890.941839:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829890.941842:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829890.941843:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941845:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829890.941847:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327389184/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941858:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829890.941860:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829890.941862:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829890.941864:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829890.941865:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829890.941866:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829890.941868:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829890.941869:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:0.0:1787829890.941871:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829890.941872:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829890.941873:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:0.0:1787829890.941876:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829890.941878:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941879:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829890.941883:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327389312/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829890.941889:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829890.941891:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829890.941892:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829890.941895:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829890.941896:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829890.941897:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829890.941899:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829890.941901:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:0.0:1787829890.941903:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829890.941904:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829890.941906:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:0.0:1787829890.941908:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829890.941909:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941911:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829890.941919:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829890.941922:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829890.941924:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829890.941926:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829891.452738:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829891.452743:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829891.452745:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829891.452749:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829891.452752:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829891.452755:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829891.452757:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829891.452759:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829891.452761:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829891.772780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829891.772787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829891.772794:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829891.772796:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829891.772800:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829891.772802:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829891.772852:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829891.772855:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829891.772858:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829892.476766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829892.476772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829892.476774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829892.476777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829892.476780:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829892.476783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.476785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829892.476787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.476789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829892.796779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829892.796785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829892.796790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829892.796792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829892.796796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829892.796797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829892.796821:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829892.796824:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829892.796827:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829892.988847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829892.988865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829892.988868:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829892.988871:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829892.988875:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829892.988881:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4040
00000400:00000200:1.0:1787829892.988890:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829892.988895:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829892.988898:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829892.988901:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.988904:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.988907:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.988909:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829892.988912:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829892.988914:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829892.988917:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829892.988920:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829892.988923:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829892.988932:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829892.988935:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.988938:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.988940:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.988943:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829892.988946:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829892.988950:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829892.988953:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829892.988956:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829892.988959:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829892.988963:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.988965:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829892.988968:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829892.988975:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829892.988979:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829892.988983:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829892.988985:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.988988:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829892.988991:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829892.988995:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829892.989004:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829892.989010:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829892.989020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829892.989030:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829892.989035:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829892.989040:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829892.989068:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829892.989072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829892.989075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829892.989077:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829892.989078:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829892.989081:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829892.989085:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4040
00000400:00000200:1.0:1787829892.989090:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829892.989093:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829892.989095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829892.989098:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989101:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.989103:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989106:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829892.989109:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829892.989110:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829892.989113:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829892.989116:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829892.989118:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829892.989120:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829892.989123:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989124:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.989126:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989128:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829892.989130:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829892.989133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989135:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829892.989137:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829892.989144:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829892.989146:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989148:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829892.989152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829892.989155:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989157:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829892.989161:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000020:00000040:1.0:1787829892.989164:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829892.989166:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829892.989168:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989170:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829892.989172:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829892.989174:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829892.989178:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829892.989182:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829892.989206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829892.989214:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829892.989216:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829892.989220:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829892.989227:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829892.989230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829892.989233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829892.989235:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829892.989236:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829892.989238:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829892.989240:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4040
00000400:00000200:1.0:1787829892.989244:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829892.989247:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829892.989248:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829892.989251:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989253:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.989255:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989256:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829892.989258:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829892.989259:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829892.989261:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829892.989263:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829892.989265:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829892.989268:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829892.989270:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989272:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.989274:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989276:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829892.989278:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829892.989280:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989283:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829892.989285:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829892.989287:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829892.989290:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989292:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829892.989294:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829892.989296:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989298:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829892.989301:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829892.989303:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829892.989306:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829892.989307:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989309:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829892.989310:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829892.989312:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829892.989316:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829892.989319:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829892.989323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829892.989330:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829892.989332:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829892.989336:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829892.989342:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829892.989346:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829892.989349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829892.989351:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829892.989353:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829892.989356:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829892.989357:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4040
00000400:00000200:1.0:1787829892.989361:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829892.989363:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829892.989366:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829892.989369:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989371:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.989373:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989375:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829892.989377:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829892.989379:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829892.989380:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829892.989382:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829892.989384:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829892.989386:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829892.989389:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989391:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829892.989393:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989395:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829892.989397:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829892.989400:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829892.989402:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829892.989404:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829892.989406:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829892.989408:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989410:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829892.989413:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829892.989415:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989417:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829892.989421:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829892.989422:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829892.989424:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829892.989425:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829892.989427:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829892.989430:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829892.989432:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829892.989435:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:1.0:1787829892.989438:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829892.989443:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829892.989452:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829892.989454:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829892.989458:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829892.989465:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829892.989470:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4050)
00000100:00000001:0.0:1787829892.989537:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829892.989543:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829892.989545:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829892.989552:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989560:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829892.989562:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.989566:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829892.989568:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829892.989570:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829892.989573:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327389568:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829892.989577:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829892.989581:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829892.989583:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.989585:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829892.989590:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829892.989592:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829892.989598:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829892.989602:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829892.989608:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829892.989614:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389568, portal 25
00000100:00000001:0.0:1787829892.989617:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829892.989619:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:0.0:1787829892.989624:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989631:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829892.989635:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829892.989638:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829892.989642:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327389568, offset 0
00000400:00000200:0.0:1787829892.989649:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829892.989684:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.989690:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829892.989696:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829892.989701:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829892.989705:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829892.989709:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829892.989713:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40815:40815:256:4294967295] 192.168.203.116@tcp LPNI seq info [40815:40815:8:4294967295]
00000400:00000200:0.0:1787829892.989717:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.989727:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829892.989734:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.989739:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829892.989745:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829892.989750:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829892.989756:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829892.989774:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829892.989778:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.989781:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829892.989786:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829892.989790:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829892.989796:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829892.989798:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829892.989802:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829892.989806:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989816:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.989819:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989827:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.989828:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829892.989834:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829892.989838:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829892.989842:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298713773
00000100:00000010:0.0:1787829892.989846:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc5a0.
00000100:00000001:0.0:1787829892.989853:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829892.989858:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298713773
00000100:00000010:0.0:1787829892.989861:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccd20.
00000100:00000001:0.0:1787829892.989865:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829892.989868:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829892.989871:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.989873:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829892.989879:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892]  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829892.989897:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829892.989898:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829892.989900:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829892.989904:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829892.989906:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829892.989908:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829892.989912:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829892.989921:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989928:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829892.989929:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.989932:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.989934:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829892.989938:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989946:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829892.989951:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.989957:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829892.989960:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.989962:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.989963:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.989965:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829892.989967:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829892.989970:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829892.989974:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829892.989977:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829892.989982:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829892.989991:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327389568:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829892.989996:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829892.990000:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990006:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829892.990007:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.990010:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829892.990012:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829892.990014:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829892.990017:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389696:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829892.990020:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829892.990022:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829892.990024:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.990026:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829892.990029:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829892.990031:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829892.990035:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829892.990039:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829892.990042:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829892.990046:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389696, portal 10
00000100:00000001:0.0:1787829892.990048:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829892.990049:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829892.990056:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990061:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829892.990064:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829892.990067:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829892.990070:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327389696, offset 0
00000400:00000200:0.0:1787829892.990074:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829892.990079:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.990082:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829892.990085:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829892.990088:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829892.990090:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829892.990093:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829892.990096:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40816:40816:256:4294967295] 192.168.203.116@tcp LPNI seq info [40816:40816:8:4294967295]
00000400:00000200:0.0:1787829892.990100:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.990107:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829892.990112:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.990115:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829892.990119:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829892.990122:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829892.990126:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829892.990129:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829892.990133:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.990135:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829892.990138:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829892.990141:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829892.990145:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829892.990146:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829892.990149:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829892.990152:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990160:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.990163:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990169:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.990170:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829892.990173:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829892.990177:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829892.990180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829892.990183:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298713099
00000100:00000001:0.0:1787829892.990198:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298713773
00000100:00000001:0.0:1787829892.990200:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298713099
00000100:00000001:0.0:1787829892.990202:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298713773
00000100:00000001:0.0:1787829892.990204:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829892.990206:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.990208:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829892.990212:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892]  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829892.990219:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829892.990221:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829892.990223:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829892.990226:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829892.990227:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829892.990229:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829892.990232:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829892.990241:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990248:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829892.990249:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990251:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.990252:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829892.990255:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990262:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829892.990266:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990273:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829892.990276:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990278:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.990279:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990281:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829892.990283:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829892.990286:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829892.990289:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829892.990292:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829892.990296:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829892.990304:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389696:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829892.990308:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829892.990311:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990317:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829892.990324:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.990326:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829892.990328:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829892.990330:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829892.990333:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389824:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829892.990337:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829892.990339:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829892.990340:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.990342:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829892.990346:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829892.990347:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829892.990350:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829892.990353:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829892.990356:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829892.990359:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389824, portal 4
00000100:00000001:0.0:1787829892.990362:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829892.990363:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829892.990368:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990375:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829892.990377:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829892.990379:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829892.990382:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327389824, offset 0
00000400:00000200:0.0:1787829892.990390:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829892.990394:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.990398:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829892.990401:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829892.990403:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829892.990405:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829892.990407:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829892.990411:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40817:40817:256:4294967295] 192.168.203.116@tcp LPNI seq info [40817:40817:8:4294967295]
00000400:00000200:0.0:1787829892.990415:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.990421:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829892.990425:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.990429:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829892.990433:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829892.990436:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829892.990440:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829892.990442:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829892.990445:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.990447:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829892.990450:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829892.990454:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829892.990457:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829892.990459:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829892.990462:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829892.990466:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990480:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.990484:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990490:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.990491:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829892.990495:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829892.990497:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829892.990501:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829892.990503:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298713106
00000100:00000001:0.0:1787829892.990506:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298713773
00000100:00000001:0.0:1787829892.990509:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298713106
00000100:00000001:0.0:1787829892.990511:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298713773
00000100:00000001:0.0:1787829892.990513:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829892.990515:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.990517:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829892.990522:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892]  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829892.990530:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829892.990532:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829892.990533:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829892.990536:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829892.990538:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829892.990540:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829892.990543:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829892.990552:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990558:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829892.990560:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990562:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.990563:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829892.990567:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990575:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829892.990579:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990586:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829892.990589:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990591:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.990592:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990594:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829892.990595:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829892.990598:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829892.990601:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829892.990603:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829892.990607:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829892.990616:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389824:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829892.990620:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829892.990623:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990630:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829892.990631:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.990634:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829892.990635:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829892.990638:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829892.990641:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389952:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829892.990644:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829892.990646:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829892.990648:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829892.990650:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829892.990653:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829892.990685:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829892.990689:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829892.990692:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829892.990696:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829892.990699:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327389952, portal 4
00000100:00000001:0.0:1787829892.990701:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829892.990702:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:0.0:1787829892.990709:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990715:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829892.990717:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829892.990720:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829892.990723:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327389952, offset 0
00000400:00000200:0.0:1787829892.990727:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829892.990731:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.990735:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829892.990738:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829892.990741:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829892.990743:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829892.990746:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829892.990749:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40818:40818:256:4294967295] 192.168.203.116@tcp LPNI seq info [40818:40818:8:4294967295]
00000400:00000200:0.0:1787829892.990754:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829892.990760:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829892.990766:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.990770:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829892.990775:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829892.990778:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829892.990782:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829892.990785:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829892.990789:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829892.990791:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829892.990794:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829892.990797:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829892.990801:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829892.990803:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829892.990805:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829892.990810:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990819:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.990823:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990830:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.990831:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829892.990834:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829892.990836:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829892.990838:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829892.990840:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298713112
00000100:00000001:0.0:1787829892.990841:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298713773
00000100:00000001:0.0:1787829892.990843:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298713112
00000100:00000001:0.0:1787829892.990850:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298713773
00000100:00000001:0.0:1787829892.990852:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829892.990853:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.990854:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829892.990858:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892]  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829892.990866:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829892.990868:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829892.990870:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829892.990872:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829892.990874:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829892.990876:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829892.990880:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829892.990889:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990896:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829892.990898:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990900:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.990902:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829892.990906:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990913:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829892.990918:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990927:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829892.990929:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990931:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829892.990932:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990934:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829892.990936:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829892.990938:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829892.990943:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829892.990945:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829892.990948:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829892.990962:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327389952:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829892.990966:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.990968:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.990971:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327389568/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.990978:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829892.990980:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829892.990983:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532c00.
02000000:00000001:0.0:1787829892.990986:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829892.990989:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829892.990990:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829892.990992:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829892.990994:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829892.990998:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829892.991000:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829892.991002:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829892.991005:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829892.991006:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.991008:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.991011:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327389696/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.991018:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829892.991019:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829892.991020:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829892.991022:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829892.991023:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829892.991025:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829892.991026:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829892.991028:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829892.991030:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829892.991031:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829892.991032:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829892.991034:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829892.991036:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.991037:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.991040:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327389824/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.991045:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829892.991046:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829892.991048:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829892.991050:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829892.991051:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829892.991052:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829892.991053:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829892.991054:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829892.991056:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829892.991057:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829892.991058:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829892.991060:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829892.991062:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.991063:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829892.991067:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327389952/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829892.991072:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829892.991073:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829892.991075:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829892.991077:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829892.991078:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829892.991080:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829892.991081:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829892.991083:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829892.991084:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829892.991085:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829892.991087:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829892.991089:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829892.991090:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.991092:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829892.991100:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829892.991103:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829892.991105:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829892.991107:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829893.501273:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829893.501280:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829893.501282:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829893.501288:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829893.501292:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829893.501296:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829893.501299:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829893.501302:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829893.501305:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829893.692779:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298713773, at 4298713792
00000100:00000040:3.0:1787829893.692779:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298713773, at 4298713792
00000100:00000010:2.0:1787829893.692796:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc5a0.
00000100:00000010:3.0:1787829893.692799:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccd20.
00010000:00000001:0.0:1787829893.820771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829893.820778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829893.820785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829893.820787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829893.820790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829893.820791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829893.820838:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829893.820841:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829893.820844:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829894.524804:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829894.524812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829894.524814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829894.524819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829894.524823:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829894.524827:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829894.524830:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829894.524833:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829894.524836:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829894.844791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829894.844799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829894.844806:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829894.844808:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829894.844811:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829894.844813:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829894.844873:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829894.844877:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829894.844881:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829895.548773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829895.548781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829895.548784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829895.548798:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829895.548802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829895.548807:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829895.548810:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829895.548813:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829895.548816:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868769:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829895.868778:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829895.868780:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829895.868784:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868788:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829895.868791:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868793:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868795:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868797:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829895.868799:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829895.868800:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829895.868801:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868802:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829895.868803:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868805:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868806:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829895.868808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829895.868809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829895.868825:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829895.868827:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829895.868829:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829896.572754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829896.572763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829896.572766:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829896.572771:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829896.572775:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829896.572778:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829896.572781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829896.572783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829896.572786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829896.892768:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829896.892775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829896.892780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829896.892782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829896.892787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829896.892789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829896.892833:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829896.892836:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829896.892839:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829897.596789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829897.596797:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829897.596798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829897.596803:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829897.596806:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829897.596810:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829897.596813:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829897.596816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829897.596821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829897.916763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829897.916769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829897.916774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829897.916776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829897.916779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829897.916780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829897.916857:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829897.916864:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829897.916876:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829898.108791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829898.108803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829898.108806:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829898.108811:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829898.108817:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829898.108831:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4045
00000400:00000200:2.0:1787829898.108842:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829898.108849:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829898.108854:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829898.108858:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.108862:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.108864:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.108868:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829898.108873:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829898.108875:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829898.108878:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829898.108882:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829898.108885:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829898.108888:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829898.108892:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.108894:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.108896:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.108898:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829898.108902:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829898.108907:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829898.108910:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829898.108913:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829898.108917:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829898.108920:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829898.108923:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829898.108926:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829898.108937:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829898.108941:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829898.108954:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829898.108956:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829898.108959:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829898.108962:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829898.108968:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829898.108976:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829898.108984:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829898.108995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829898.109007:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829898.109012:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829898.109017:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829898.109042:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829898.109045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829898.109049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829898.109051:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829898.109052:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829898.109056:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829898.109060:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4045
00000400:00000200:2.0:1787829898.109065:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829898.109069:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829898.109072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829898.109078:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109081:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.109085:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109088:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829898.109091:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829898.109092:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829898.109094:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829898.109098:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829898.109099:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787829898.109101:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829898.109107:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829898.109108:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829898.109114:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829898.109114:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109123:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.109129:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109136:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829898.109138:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829898.109142:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829898.109166:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109175:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.109178:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787829898.109180:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829898.109182:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.109189:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829898.109189:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829898.109193:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829898.109194:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.109203:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829898.109203:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829898.109213:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1787829898.109214:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327390080:192.168.203.116@tcp:250:kworker.0
00000008:00000001:2.0:1787829898.109224:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.109226:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787829898.109230:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.109234:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829898.109240:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829898.109240:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829898.109248:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829898.109249:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787829898.109256:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829898.109257:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829898.109259:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:0.0:1787829898.109261:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:2.0:1787829898.109262:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829898.109267:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829898.109267:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829898.109280:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000400:00000200:0.0:1787829898.109288:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829898.109292:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829898.109295:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829898.109303:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829898.109310:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390080, portal 25
00000100:00100000:2.0:1787829898.109310:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829898.109317:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829898.109321:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000100:00000001:2.0:1787829898.109333:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1787829898.109335:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829898.109337:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829898.109354:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.109362:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829898.109368:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829898.109370:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829898.109376:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327390080, offset 0
00000400:00000200:0.0:1787829898.109381:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109393:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.109394:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829898.109401:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829898.109406:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829898.109409:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829898.109412:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829898.109415:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787829898.109418:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829898.109424:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787829898.109427:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4045
00000400:00000200:0.0:1787829898.109433:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:2.0:1787829898.109438:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.109442:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829898.109446:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:2.0:1787829898.109457:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829898.109459:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40819:40819:256:4294967295] 192.168.203.116@tcp LPNI seq info [40819:40819:8:4294967295]
00000100:00080000:2.0:1787829898.109463:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829898.109475:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829898.109476:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829898.109483:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.109488:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109497:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829898.109503:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829898.109504:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787829898.109506:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829898.109512:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829898.109516:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:0.0:1787829898.109521:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829898.109524:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:0.0:1787829898.109529:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000020:00000001:2.0:1787829898.109529:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829898.109540:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829898.109547:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829898.109547:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.109554:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829898.109557:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787829898.109559:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829898.109570:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.1:1787829898.109577:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787829898.109578:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109585:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829898.109586:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
02000000:00000001:2.0:1787829898.109589:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829898.109595:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:0.0:1787829898.109596:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829898.109602:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829898.109603:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000100:00000001:2.0:1787829898.109614:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829898.109619:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:2.0:1787829898.109620:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829898.109627:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.109631:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829898.109632:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.109638:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00000010:2.0:1787829898.109641:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000400:00000010:0.0:1787829898.109643:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000020:00000040:2.0:1787829898.109647:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:0.0:1787829898.109650:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787829898.109652:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:0.0:1787829898.109687:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829898.109694:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829898.109701:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829898.109704:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829898.109710:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829898.109721:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:0.0:1787829898.109722:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829898.109729:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829898.109731:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829898.109743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829898.109749:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.109760:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787829898.109767:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829898.109768:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829898.109772:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.109780:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829898.109781:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.109785:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298718893
00000100:00000010:0.0:1787829898.109797:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc180.
00000100:00000001:2.0:1787829898.109807:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.109813:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:2.0:1787829898.109813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829898.109821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829898.109823:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298718893
00000100:00000001:2.0:1787829898.109824:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829898.109827:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:0.0:1787829898.109830:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc660.
00000100:00000001:2.0:1787829898.109833:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829898.109837:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00080000:2.0:1787829898.109838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4045
00000400:00000200:2.0:1787829898.109847:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829898.109850:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787829898.109854:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829898.109861:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829898.109861:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829898.109869:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787829898.109871:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109880:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:0.0:1787829898.109881:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829898/real 1787829898]  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829898.109887:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109894:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829898.109902:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829898.109906:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000001:0.0:1787829898.109909:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:2.0:1787829898.109911:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000001:0.0:1787829898.109913:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:2.0:1787829898.109915:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000400:00000010:0.0:1787829898.109918:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000020:00000001:2.0:1787829898.109920:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:0.0:1787829898.109926:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000020:00000001:2.0:1787829898.109927:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000010:0.0:1787829898.109933:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000040:2.0:1787829898.109936:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829898.109939:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787829898.109941:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829898.109949:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829898.109950:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829898.109956:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829898.109963:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829898.109976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829898.109983:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829898.109989:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829898.109990:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829898.109997:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829898.110005:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110007:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787829898.110010:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829898.110012:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:2.0:1787829898.110014:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829898.110017:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00000001:2.0:1787829898.110021:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110022:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829898.110031:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829898.110037:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787829898.110040:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829898.110049:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829898.110059:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829898.110064:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110068:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829898.110069:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829898.110075:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:0.0:1787829898.110081:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829898.110081:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829898.110092:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:0.0:1787829898.110103:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787829898.110105:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110107:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110111:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.110117:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787829898.110118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829898.110123:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829898.110129:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829898.110142:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1787829898.110145:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00080000:0.0:1787829898.110162:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:2.0:1787829898.110163:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110174:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787829898.110177:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829898.110185:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829898.110207:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829898.110220:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4055)
00000100:00100000:0.0:1787829898.110221:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327390080:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829898.110237:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110239:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.110243:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327390080/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110250:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829898.110252:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829898.110255:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829898.110258:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829898.110261:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829898.110263:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829898.110265:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829898.110267:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829898.110271:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829898.110273:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829898.110275:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:0.0:1787829898.110278:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829898.110280:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110282:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110288:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829898.110289:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110292:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829898.110293:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829898.110295:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829898.110299:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110305:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829898.110307:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.110309:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829898.110311:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829898.110313:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829898.110316:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390208:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829898.110319:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829898.110321:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829898.110323:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.110324:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829898.110328:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829898.110329:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.110333:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829898.110335:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829898.110338:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829898.110341:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390208, portal 10
00000100:00000001:0.0:1787829898.110343:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829898.110344:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:0.0:1787829898.110351:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110357:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829898.110360:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829898.110362:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829898.110364:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327390208, offset 0
00000400:00000200:0.0:1787829898.110367:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829898.110372:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.110375:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829898.110377:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829898.110380:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829898.110382:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829898.110384:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829898.110386:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40820:40820:256:4294967295] 192.168.203.116@tcp LPNI seq info [40820:40820:8:4294967295]
00000400:00000200:0.0:1787829898.110390:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.110395:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829898.110399:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829898.110402:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829898.110406:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829898.110409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829898.110413:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829898.110415:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829898.110418:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829898.110420:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829898.110423:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829898.110426:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829898.110428:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829898.110430:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829898.110432:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829898.110435:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110442:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.110445:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110457:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110458:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829898.110461:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829898.110465:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829898.110468:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829898.110470:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298718219
00000100:00000001:0.0:1787829898.110473:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298718893
00000100:00000001:0.0:1787829898.110475:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298718219
00000100:00000001:0.0:1787829898.110477:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298718893
00000100:00000001:0.0:1787829898.110479:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829898.110481:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110482:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829898.110486:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829898/real 1787829898]  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829898.110492:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829898.110494:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829898.110495:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829898.110497:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829898.110499:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829898.110501:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829898.110504:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829898.110513:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110519:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829898.110521:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110522:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.110523:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829898.110527:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110534:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829898.110538:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110544:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829898.110546:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110548:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.110549:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110550:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829898.110552:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829898.110555:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829898.110558:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829898.110560:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829898.110564:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829898.110571:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390208:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829898.110575:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829898.110578:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110584:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829898.110586:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.110588:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829898.110589:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829898.110591:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829898.110594:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829898.110597:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829898.110599:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829898.110600:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.110602:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829898.110605:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829898.110607:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.110610:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829898.110612:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829898.110615:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829898.110617:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390336, portal 4
00000100:00000001:0.0:1787829898.110619:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829898.110620:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:0.0:1787829898.110625:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110631:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829898.110633:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829898.110635:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829898.110637:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327390336, offset 0
00000400:00000200:0.0:1787829898.110640:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829898.110644:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.110647:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829898.110649:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829898.110651:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829898.110654:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829898.110675:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829898.110678:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40821:40821:256:4294967295] 192.168.203.116@tcp LPNI seq info [40821:40821:8:4294967295]
00000400:00000200:0.0:1787829898.110682:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.110688:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829898.110692:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829898.110695:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829898.110699:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829898.110702:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829898.110706:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829898.110708:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829898.110711:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829898.110714:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829898.110716:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829898.110720:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829898.110722:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829898.110724:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829898.110726:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829898.110729:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110736:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.110739:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110746:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110747:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829898.110749:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829898.110752:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829898.110755:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829898.110757:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298718227
00000100:00000001:0.0:1787829898.110759:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298718893
00000100:00000001:0.0:1787829898.110761:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298718227
00000100:00000001:0.0:1787829898.110763:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298718893
00000100:00000001:0.0:1787829898.110765:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829898.110767:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.110769:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829898.110772:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829898/real 1787829898]  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829898.110780:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829898.110781:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829898.110783:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829898.110785:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829898.110786:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829898.110788:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829898.110791:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829898.110799:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110805:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829898.110806:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110807:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.110809:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829898.110812:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110819:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829898.110822:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110828:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829898.110830:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110832:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.110833:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.110835:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829898.110836:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829898.110838:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829898.110840:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829898.110842:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829898.110846:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829898.110854:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390336:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829898.110857:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829898.110861:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110867:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829898.110868:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.110871:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829898.110872:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829898.110874:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829898.110877:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390464:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829898.110879:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829898.110881:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829898.110883:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829898.110884:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829898.110887:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829898.110889:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829898.110891:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829898.110893:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829898.110896:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829898.110898:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390464, portal 4
00000100:00000001:0.0:1787829898.110900:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829898.110901:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:0.0:1787829898.110907:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110913:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829898.110916:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829898.110917:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829898.110919:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327390464, offset 0
00000400:00000200:0.0:1787829898.110923:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829898.110926:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.110929:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829898.110931:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829898.110934:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829898.110936:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829898.110938:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829898.110941:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40822:40822:256:4294967295] 192.168.203.116@tcp LPNI seq info [40822:40822:8:4294967295]
00000400:00000200:0.0:1787829898.110944:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829898.110950:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829898.110954:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829898.110956:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829898.110960:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829898.110963:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829898.110966:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829898.110968:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829898.110972:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829898.110974:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829898.110976:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829898.110980:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829898.110982:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829898.110984:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829898.110986:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829898.110989:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.110995:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.110998:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111005:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.111007:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829898.111009:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829898.111011:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829898.111014:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829898.111016:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298718232
00000100:00000001:0.0:1787829898.111018:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298718893
00000100:00000001:0.0:1787829898.111020:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298718232
00000100:00000001:0.0:1787829898.111022:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298718893
00000100:00000001:0.0:1787829898.111024:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829898.111026:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.111028:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829898.111031:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829898/real 1787829898]  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829898.111039:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829898.111040:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829898.111042:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829898.111044:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829898.111045:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829898.111047:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829898.111049:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829898.111057:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111064:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829898.111065:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111067:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.111068:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829898.111071:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111078:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829898.111081:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111087:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829898.111090:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111091:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829898.111092:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111094:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829898.111095:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829898.111097:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829898.111101:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829898.111103:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829898.111106:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829898.111113:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390464:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829898.111117:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111119:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.111121:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327390208/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111127:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829898.111129:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829898.111131:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829898.111133:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829898.111134:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829898.111135:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829898.111137:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829898.111139:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:0.0:1787829898.111142:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829898.111143:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829898.111144:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:0.0:1787829898.111147:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829898.111160:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111162:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.111165:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327390336/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111171:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829898.111173:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829898.111174:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829898.111177:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829898.111178:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829898.111179:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829898.111181:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829898.111183:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829898.111185:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829898.111186:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829898.111187:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:0.0:1787829898.111190:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829898.111191:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111192:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829898.111195:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327390464/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829898.111201:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829898.111202:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829898.111203:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829898.111205:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829898.111206:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829898.111207:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829898.111208:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829898.111209:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829898.111211:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829898.111212:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829898.111213:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:0.0:1787829898.111215:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829898.111216:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111218:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.111221:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829898.111222:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.111224:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829898.111225:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829898.620772:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829898.620778:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829898.620779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829898.620784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829898.620787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829898.620790:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.620792:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829898.620802:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829898.620805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829898.812727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298718893, at 4298718912
00000100:00000040:3.0:1787829898.812735:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298718893, at 4298718912
00000100:00000010:2.0:1787829898.812741:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc180.
00000100:00000010:3.0:1787829898.812744:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc660.
00010000:00000001:0.0:1787829898.940763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829898.940769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829898.940772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829898.940773:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829898.940776:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829898.940777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829898.940798:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829898.940800:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829898.940802:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829899.644754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829899.644758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829899.644760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829899.644763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829899.644766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829899.644769:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829899.644771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829899.644772:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829899.644774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829899.964771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829899.964778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829899.964786:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829899.964787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829899.964791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829899.964792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829899.964848:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829899.964853:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829899.964857:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829900.668760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829900.668766:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829900.668769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829900.668773:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829900.668777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829900.668780:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829900.668783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829900.668785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829900.668788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829900.988803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829900.988812:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829900.988819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829900.988821:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829900.988825:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829900.988829:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829900.988832:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829900.988836:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829900.988840:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829900.988843:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829900.988845:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829900.988848:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829900.988904:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829900.988909:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829900.988913:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829901.692748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829901.692753:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829901.692755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829901.692758:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829901.692760:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829901.692763:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829901.692765:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829901.692767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829901.692769:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829902.012755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829902.012760:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829902.012766:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829902.012767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829902.012770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829902.012772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829902.012810:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829902.012813:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829902.012817:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829902.716750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829902.716755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829902.716757:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829902.716762:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829902.716765:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829902.716768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829902.716771:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829902.716774:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829902.716777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829903.036761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829903.036765:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829903.036770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829903.036771:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829903.036773:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829903.036775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829903.036824:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829903.036827:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829903.036829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829903.228783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829903.228793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829903.228796:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829903.228799:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829903.228803:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829903.228810:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4050
00000400:00000200:1.0:1787829903.228818:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829903.228825:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829903.228828:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829903.228831:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228834:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829903.228837:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228839:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829903.228842:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829903.228845:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829903.228847:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829903.228850:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829903.228852:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829903.228855:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829903.228857:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228859:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829903.228861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228862:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829903.228865:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829903.228868:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228870:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.228872:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829903.228875:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829903.228877:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.228879:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829903.228881:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829903.228889:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829903.228892:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829903.228895:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829903.228897:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.228899:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829903.228902:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829903.228905:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829903.228912:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829903.228917:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829903.228925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829903.228934:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829903.228939:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829903.228943:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829903.228963:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829903.228966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829903.228969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829903.228970:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829903.228971:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829903.228974:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829903.228976:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4050
00000400:00000200:1.0:1787829903.228979:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829903.228982:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829903.228985:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829903.228988:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228990:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829903.228993:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.228994:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829903.228996:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787829903.228997:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829903.229000:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:0.0:1787829903.229002:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787829903.229003:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829903.229006:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829903.229007:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787829903.229009:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829903.229012:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829903.229017:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829903.229018:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.229021:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829903.229025:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.229029:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829903.229031:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829903.229035:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829903.229040:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829903.229042:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229043:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229046:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829903.229049:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829903.229054:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829903.229054:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829903.229058:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229059:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.229063:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:0.0:1787829903.229065:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327390592:192.168.203.116@tcp:250:kworker.0
00000008:00100000:1.0:1787829903.229068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829903.229072:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:1.0:1787829903.229073:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.229076:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.229078:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:1.0:1787829903.229080:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
02000000:00000001:0.0:1787829903.229081:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787829903.229085:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829903.229086:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787829903.229090:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829903.229092:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829903.229093:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000100:00000001:1.0:1787829903.229094:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829903.229097:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829903.229098:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229100:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829903.229105:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:1.0:1787829903.229107:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000400:00000010:0.0:1787829903.229111:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00000001:1.0:1787829903.229127:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829903.229128:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829903.229137:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390592, portal 25
00000100:00100000:1.0:1787829903.229137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829903.229140:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829903.229143:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000001:1.0:1787829903.229151:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1787829903.229153:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829903.229155:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829903.229164:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.229168:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829903.229174:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829903.229180:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000001:1.0:1787829903.229180:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829903.229184:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327390592, offset 0
00000100:00080000:1.0:1787829903.229185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829903.229191:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829903.229195:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829903.229195:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829903.229199:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829903.229202:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829903.229204:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829903.229207:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4050
00000400:00000200:0.0:1787829903.229213:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829903.229219:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829903.229223:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829903.229226:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829903.229229:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829903.229233:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40823:40823:256:4294967295] 192.168.203.116@tcp LPNI seq info [40823:40823:8:4294967295]
00000400:00000200:0.0:1787829903.229237:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829903.229243:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829903.229252:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829903.229253:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1787829903.229255:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829903.229262:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829903.229264:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.229270:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829903.229275:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829903.229279:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1787829903.229280:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829903.229286:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.1:1787829903.229290:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829903.229291:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.1:1787829903.229295:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000020:00000001:1.0:1787829903.229296:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829903.229299:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829903.229303:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:0.0:1787829903.229304:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.229307:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000020:00000040:1.0:1787829903.229311:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:0.0:1787829903.229313:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1787829903.229315:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829903.229320:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000020:00000001:1.0:1787829903.229321:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829903.229326:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00000040:1.0:1787829903.229328:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829903.229330:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:1.0:1787829903.229332:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829903.229337:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787829903.229337:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.229341:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000200:0.0:1787829903.229346:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829903.229346:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829903.229353:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829903.229359:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229362:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829903.229366:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1787829903.229367:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829903.229371:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829903.229373:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829903.229374:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829903.229378:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829903.229381:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.229382:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829903.229386:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829903.229391:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829903.229397:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829903.229398:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229399:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229402:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829903.229404:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829903.229408:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829903.229409:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229410:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829903.229416:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000010:1.0:1787829903.229417:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:0.0:1787829903.229420:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298724012
00000100:00000001:1.0:1787829903.229422:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829903.229427:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf60.
00000100:00100000:1.0:1787829903.229430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829903.229435:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829903.229440:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829903.229443:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298724012
00000100:00000001:1.0:1787829903.229444:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829903.229448:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccde0.
00000100:00000040:1.0:1787829903.229452:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.229458:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829903.229462:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829903.229466:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829903.229469:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787829903.229470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829903.229473:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787829903.229475:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829903.229476:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829903.229480:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829903.229481:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:0.0:1787829903.229487:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829903/real 1787829903]  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829903.229487:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4050
00000400:00000200:1.0:1787829903.229497:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829903.229504:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000001:0.0:1787829903.229505:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:1.0:1787829903.229507:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829903.229511:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000001:0.0:1787829903.229514:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1787829903.229515:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1787829903.229517:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000100:00000040:1.0:1787829903.229520:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829903.229522:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000100:00000001:1.0:1787829903.229524:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:0.0:1787829903.229526:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000020:00000001:1.0:1787829903.229528:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829903.229530:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:1.0:1787829903.229531:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829903.229535:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:0.0:1787829903.229536:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787829903.229538:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829903.229543:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829903.229548:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829903.229553:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829903.229555:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829903.229561:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829903.229565:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829903.229567:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829903.229569:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829903.229573:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829903.229574:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1787829903.229577:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829903.229579:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.229583:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829903.229583:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829903.229588:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1787829903.229592:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829903.229593:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829903.229599:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.229603:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829903.229606:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829903.229607:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:1.0:1787829903.229610:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829903.229614:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829903.229615:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787829903.229620:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829903.229625:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829903.229626:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:1.0:1787829903.229630:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829903.229631:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787829903.229633:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229634:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829903.229637:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829903.229639:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829903.229640:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229643:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.229647:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000010:1.0:1787829903.229648:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5c00.
00000100:00000001:0.0:1787829903.229649:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829903.229675:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829903.229680:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829903.229682:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829903.229687:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829903.229687:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829903.229697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829903.229702:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327390592:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829903.229712:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829903.229713:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829903.229716:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829903.229717:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829903.229722:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327390592/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829903.229726:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829903.229738:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229742:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000040:1.0:1787829903.229744:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4061)
02000000:00000001:0.0:1787829903.229746:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829903.229751:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829903.229754:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829903.229756:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829903.229757:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829903.229760:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829903.229763:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:0.0:1787829903.229767:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829903.229769:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829903.229770:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829903.229773:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829903.229775:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.229777:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.229783:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829903.229785:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229787:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829903.229788:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829903.229789:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829903.229793:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.229800:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829903.229801:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.229803:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829903.229805:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829903.229807:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829903.229810:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390720:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829903.229813:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829903.229815:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829903.229816:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.229818:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829903.229821:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829903.229822:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829903.229826:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829903.229828:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829903.229839:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829903.229842:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390720, portal 10
00000100:00000001:0.0:1787829903.229844:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829903.229845:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829903.229852:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.229858:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829903.229861:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829903.229863:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829903.229865:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327390720, offset 0
00000400:00000200:0.0:1787829903.229868:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829903.229872:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829903.229875:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829903.229878:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829903.229881:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829903.229883:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829903.229885:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829903.229889:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40824:40824:256:4294967295] 192.168.203.116@tcp LPNI seq info [40824:40824:8:4294967295]
00000400:00000200:0.0:1787829903.229892:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829903.229899:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829903.229909:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.229912:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829903.229916:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829903.229918:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787829903.229923:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829903.229925:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787829903.229929:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.229931:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829903.229934:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829903.229937:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829903.229940:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829903.229941:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829903.229944:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829903.229947:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.229955:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829903.229958:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.229965:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229967:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829903.229969:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829903.229973:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829903.229976:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829903.229978:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298723339
00000100:00000001:0.0:1787829903.229980:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298724012
00000100:00000001:0.0:1787829903.229982:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298723339
00000100:00000001:0.0:1787829903.229984:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298724012
00000100:00000001:0.0:1787829903.229986:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829903.229988:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.229990:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829903.229993:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829903/real 1787829903]  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829903.230000:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829903.230002:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829903.230004:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829903.230006:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829903.230007:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829903.230009:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829903.230012:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829903.230021:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230026:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829903.230028:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230029:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.230031:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829903.230034:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230041:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829903.230044:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230051:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829903.230053:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230055:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.230056:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230057:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829903.230059:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829903.230061:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829903.230064:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829903.230066:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829903.230070:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829903.230078:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390720:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829903.230081:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829903.230084:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230089:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829903.230091:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.230093:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829903.230094:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829903.230096:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829903.230098:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829903.230102:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829903.230103:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829903.230105:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.230107:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829903.230109:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829903.230111:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829903.230129:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829903.230131:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829903.230134:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829903.230137:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390848, portal 4
00000100:00000001:0.0:1787829903.230139:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829903.230140:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829903.230145:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230152:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829903.230154:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829903.230156:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829903.230159:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327390848, offset 0
00000400:00000200:0.0:1787829903.230162:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829903.230165:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829903.230168:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829903.230171:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829903.230173:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829903.230176:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829903.230178:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829903.230181:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40825:40825:256:4294967295] 192.168.203.116@tcp LPNI seq info [40825:40825:8:4294967295]
00000400:00000200:0.0:1787829903.230185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829903.230191:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829903.230195:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.230197:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829903.230200:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829903.230203:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787829903.230207:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829903.230210:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787829903.230214:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.230215:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829903.230218:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829903.230221:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829903.230224:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829903.230226:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829903.230228:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829903.230231:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230238:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829903.230241:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230247:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.230249:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829903.230252:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829903.230254:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829903.230257:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829903.230259:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298723346
00000100:00000001:0.0:1787829903.230261:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298724012
00000100:00000001:0.0:1787829903.230263:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298723346
00000100:00000001:0.0:1787829903.230265:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298724012
00000100:00000001:0.0:1787829903.230267:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829903.230269:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.230271:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829903.230274:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829903/real 1787829903]  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829903.230281:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829903.230282:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829903.230284:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829903.230286:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829903.230288:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829903.230290:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829903.230292:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829903.230300:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230306:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829903.230308:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230309:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.230311:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829903.230314:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230320:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829903.230324:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230331:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829903.230333:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230334:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.230335:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230337:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829903.230339:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829903.230341:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829903.230344:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829903.230346:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829903.230349:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829903.230357:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390848:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829903.230360:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829903.230363:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230370:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829903.230371:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.230373:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829903.230374:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829903.230376:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829903.230379:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390976:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829903.230382:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829903.230383:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829903.230384:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829903.230386:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829903.230389:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829903.230390:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829903.230393:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829903.230395:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829903.230398:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829903.230400:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327390976, portal 4
00000100:00000001:0.0:1787829903.230402:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829903.230403:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:0.0:1787829903.230408:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230414:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829903.230416:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829903.230418:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829903.230420:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327390976, offset 0
00000400:00000200:0.0:1787829903.230423:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829903.230426:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829903.230429:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829903.230432:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829903.230434:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829903.230436:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829903.230438:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829903.230441:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40826:40826:256:4294967295] 192.168.203.116@tcp LPNI seq info [40826:40826:8:4294967295]
00000400:00000200:0.0:1787829903.230445:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829903.230450:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829903.230454:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.230457:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829903.230460:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829903.230462:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787829903.230466:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829903.230468:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787829903.230472:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829903.230474:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829903.230476:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829903.230480:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829903.230482:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829903.230484:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829903.230486:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829903.230489:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230496:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829903.230499:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230505:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.230507:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829903.230509:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829903.230512:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829903.230514:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829903.230516:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298723351
00000100:00000001:0.0:1787829903.230518:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298724012
00000100:00000001:0.0:1787829903.230520:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298723351
00000100:00000001:0.0:1787829903.230522:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298724012
00000100:00000001:0.0:1787829903.230524:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829903.230525:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.230527:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829903.230530:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829903/real 1787829903]  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829903.230537:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829903.230539:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829903.230540:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829903.230542:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829903.230543:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829903.230545:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829903.230548:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829903.230556:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230567:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829903.230569:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230571:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.230572:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829903.230575:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230581:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829903.230585:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230592:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829903.230594:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230595:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829903.230596:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230598:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829903.230599:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829903.230601:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829903.230604:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829903.230606:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829903.230610:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829903.230617:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327390976:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829903.230620:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230621:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829903.230624:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327390720/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230630:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829903.230631:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829903.230633:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829903.230635:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829903.230637:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829903.230638:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829903.230640:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829903.230641:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829903.230643:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829903.230645:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829903.230646:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829903.230649:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829903.230650:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230651:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829903.230668:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327390848/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230675:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829903.230676:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829903.230678:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829903.230680:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829903.230681:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829903.230682:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829903.230684:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829903.230685:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:0.0:1787829903.230687:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829903.230689:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829903.230690:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829903.230692:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829903.230693:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230694:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829903.230697:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327390976/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829903.230703:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829903.230704:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829903.230706:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829903.230707:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829903.230708:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829903.230709:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829903.230711:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829903.230713:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5c00.
02000000:00000001:0.0:1787829903.230715:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829903.230716:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829903.230717:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829903.230719:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829903.230720:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230722:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.230724:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829903.230725:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.230727:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829903.230728:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829903.740760:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829903.740767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829903.740769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829903.740774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829903.740777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829903.740780:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.740781:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829903.740783:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829903.740786:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829903.932748:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298724012, at 4298724032
00000100:00000040:2.0:1787829903.932751:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298724012, at 4298724032
00000100:00000010:3.0:1787829903.932765:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccde0.
00000100:00000010:2.0:1787829903.932767:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf60.
00010000:00000001:0.0:1787829904.060745:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829904.060751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829904.060756:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829904.060757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829904.060760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829904.060761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829904.060834:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829904.060839:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829904.060842:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829904.764729:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829904.764735:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829904.764737:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829904.764741:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829904.764744:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829904.764746:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829904.764748:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829904.764750:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829904.764751:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829905.084773:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829905.084780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829905.084785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829905.084787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829905.084789:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829905.084790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829905.084807:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829905.084809:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829905.084811:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829905.788808:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829905.788818:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829905.788820:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829905.788826:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829905.788830:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829905.788843:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829905.788846:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829905.788849:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829905.788853:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108786:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829906.108798:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829906.108802:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829906.108807:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108813:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829906.108816:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108819:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108823:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108829:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829906.108831:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829906.108833:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829906.108835:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108837:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829906.108840:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108842:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108846:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829906.108851:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829906.108853:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829906.108915:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829906.108924:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829906.108927:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829906.812716:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829906.812722:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829906.812723:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829906.812727:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829906.812730:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829906.812732:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829906.812734:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829906.812735:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829906.812738:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829907.132747:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829907.132752:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829907.132757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829907.132758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829907.132760:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829907.132761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829907.132822:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829907.132826:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829907.132829:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829907.836763:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829907.836770:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829907.836772:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829907.836776:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829907.836779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829907.836783:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829907.836785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829907.836788:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829907.836790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829908.156769:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829908.156777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829908.156782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829908.156784:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829908.156787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829908.156788:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829908.156806:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829908.156808:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829908.156810:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829908.348759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829908.348768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829908.348771:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829908.348774:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829908.348777:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829908.348783:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4056
00000400:00000200:2.0:1787829908.348790:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829908.348796:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829908.348798:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829908.348801:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348804:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829908.348806:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348808:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829908.348811:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829908.348819:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829908.348822:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829908.348825:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829908.348827:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829908.348829:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829908.348832:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348834:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829908.348835:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348838:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829908.348840:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829908.348843:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348845:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829908.348848:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829908.348851:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829908.348853:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.348855:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829908.348856:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829908.348865:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829908.348868:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829908.348871:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829908.348873:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.348876:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829908.348878:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829908.348881:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829908.348887:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829908.348893:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829908.348900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829908.348909:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829908.348913:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829908.348917:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829908.348934:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829908.348936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829908.348939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829908.348941:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829908.348942:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829908.348945:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829908.348947:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4056
00000400:00000200:2.0:1787829908.348950:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829908.348954:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829908.348956:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829908.348959:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348961:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829908.348964:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348966:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829908.348968:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829908.348970:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829908.348971:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829908.348973:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787829908.348974:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829908.348978:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787829908.348978:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829908.348982:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787829908.348983:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829908.348989:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829908.348989:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.348992:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829908.348997:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349001:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829908.349003:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787829908.349005:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829908.349006:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.349011:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:2.0:1787829908.349012:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:0.0:1787829908.349015:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829908.349017:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.349018:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829908.349021:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1787829908.349024:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327391104:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787829908.349027:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829908.349028:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829908.349031:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829908.349033:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.349034:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829908.349038:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829908.349040:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829908.349041:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.349045:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829908.349045:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:0.0:1787829908.349050:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000008:00100000:2.0:1787829908.349051:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1787829908.349054:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000008:00000001:2.0:1787829908.349055:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349060:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829908.349062:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391104, portal 25
00000100:00000010:2.0:1787829908.349064:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000100:00000001:0.0:1787829908.349065:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829908.349068:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000020:00000040:2.0:1787829908.349069:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829908.349073:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:0.0:1787829908.349074:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829908.349086:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349088:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829908.349090:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829908.349092:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829908.349096:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829908.349102:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349105:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829908.349111:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00100000:2.0:1787829908.349112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829908.349114:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829908.349120:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327391104, offset 0
00000100:00000001:2.0:1787829908.349127:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829908.349131:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349137:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.349141:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829908.349145:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829908.349151:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:2.0:1787829908.349162:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829908.349166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829908.349167:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787829908.349172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1787829908.349173:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787829908.349178:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829908.349180:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1787829908.349181:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829908.349184:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349186:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829908.349191:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4056
00000400:00000200:0.0:1787829908.349192:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40827:40827:256:4294967295] 192.168.203.116@tcp LPNI seq info [40827:40827:8:4294967295]
00000400:00000200:2.0:1787829908.349198:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.349201:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829908.349204:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829908.349207:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829908.349211:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349215:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829908.349217:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1787829908.349221:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:0.0:1787829908.349227:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349227:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829908.349232:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:0.0:1787829908.349233:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000020:00000040:2.0:1787829908.349235:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829908.349238:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.0:1787829908.349242:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787829908.349242:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:0.0:1787829908.349246:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1787829908.349247:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829908.349254:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.1:1787829908.349254:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:2.0:1787829908.349259:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349260:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829908.349261:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349263:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829908.349264:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829908.349269:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:0.1:1787829908.349270:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000100:00000001:2.0:1787829908.349273:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829908.349278:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829908.349278:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:0.0:1787829908.349280:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
02000000:00000001:2.0:1787829908.349283:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829908.349285:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787829908.349289:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.349291:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829908.349294:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829908.349296:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000008:00100000:2.0:1787829908.349298:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1787829908.349299:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829908.349303:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:2.0:1787829908.349304:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829908.349309:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829908.349309:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829908.349316:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000020:00000040:2.0:1787829908.349319:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:0.0:1787829908.349320:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787829908.349323:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:0.0:1787829908.349325:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829908.349325:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349330:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829908.349333:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829908.349336:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349337:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349339:0:2346:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1787829908.349340:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:0.0:1787829908.349344:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349345:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.349349:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829908.349352:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298729132
00000100:00000010:0.0:1787829908.349356:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc6c0.
00000100:00100000:2.0:1787829908.349357:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829908.349363:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829908.349369:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298729132
00000100:00000001:2.0:1787829908.349369:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:0.0:1787829908.349373:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccea0.
00000100:00000001:2.0:1787829908.349374:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349376:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000040:2.0:1787829908.349381:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349386:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829908.349392:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349393:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349396:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:2.0:1787829908.349398:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829908.349403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00100000:0.0:1787829908.349405:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829908/real 1787829908]  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829908.349406:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829908.349409:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829908.349414:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829908.349419:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4056
00000400:00000200:2.0:1787829908.349431:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000001:0.0:1787829908.349433:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1787829908.349436:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000001:0.0:1787829908.349437:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1787829908.349439:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000010:0.0:1787829908.349441:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000100:00000040:2.0:1787829908.349442:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349447:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829908.349448:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000100:00000040:2.0:1787829908.349449:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349453:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:0.0:1787829908.349454:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000020:00000001:2.0:1787829908.349455:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829908.349458:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829908.349460:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:2.0:1787829908.349460:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829908.349464:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829908.349466:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829908.349468:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829908.349468:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829908.349474:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349479:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829908.349482:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349485:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829908.349487:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000200:0.0:1787829908.349490:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829908.349491:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829908.349496:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829908.349499:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829908.349502:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829908.349504:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349505:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829908.349507:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829908.349509:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829908.349510:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829908.349514:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
00000100:00000001:0.0:1787829908.349517:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:2.0:1787829908.349518:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829908.349521:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787829908.349521:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829908.349524:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349527:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787829908.349527:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829908.349529:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829908.349531:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787829908.349532:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829908.349538:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787829908.349540:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829908.349545:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829908.349552:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829908.349555:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829908.349561:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829908.349563:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829908.349566:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829908.349568:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829908.349596:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349597:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829908.349602:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787829908.349602:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4066)
00000100:00000001:0.0:1787829908.349604:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.349607:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349610:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829908.349612:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829908.349615:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829908.349620:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829908.349623:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829908.349627:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829908.349636:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327391104:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829908.349640:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349642:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.349645:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327391104/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349652:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829908.349654:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829908.349679:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829908.349683:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829908.349684:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829908.349686:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829908.349688:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829908.349691:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:0.0:1787829908.349694:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829908.349696:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829908.349698:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:0.0:1787829908.349701:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829908.349702:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349704:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349710:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829908.349712:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349714:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829908.349715:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829908.349717:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829908.349721:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349727:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829908.349728:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.349730:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829908.349732:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829908.349734:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829908.349737:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391232:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829908.349740:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829908.349742:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829908.349743:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.349745:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829908.349748:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829908.349749:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.349753:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829908.349755:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829908.349758:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829908.349761:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391232, portal 10
00000100:00000001:0.0:1787829908.349762:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829908.349763:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:0.0:1787829908.349770:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349775:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829908.349777:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829908.349779:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829908.349781:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327391232, offset 0
00000400:00000200:0.0:1787829908.349785:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829908.349789:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.349792:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829908.349802:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829908.349805:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829908.349807:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829908.349809:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829908.349812:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40828:40828:256:4294967295] 192.168.203.116@tcp LPNI seq info [40828:40828:8:4294967295]
00000400:00000200:0.0:1787829908.349817:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.349823:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829908.349827:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829908.349830:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829908.349834:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829908.349837:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829908.349842:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829908.349844:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829908.349848:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829908.349849:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829908.349852:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829908.349855:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829908.349858:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829908.349860:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829908.349862:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829908.349865:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349873:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.349876:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349882:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349883:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829908.349886:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829908.349890:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829908.349893:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829908.349895:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298728459
00000100:00000001:0.0:1787829908.349897:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298729132
00000100:00000001:0.0:1787829908.349899:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298728459
00000100:00000001:0.0:1787829908.349901:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298729132
00000100:00000001:0.0:1787829908.349903:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829908.349905:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.349907:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829908.349911:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829908/real 1787829908]  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829908.349918:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829908.349919:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829908.349921:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829908.349924:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829908.349926:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829908.349928:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829908.349931:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829908.349939:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349944:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829908.349945:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349947:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.349948:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829908.349952:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349958:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829908.349961:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.349967:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829908.349970:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349971:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.349972:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.349974:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829908.349976:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829908.349979:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829908.349982:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829908.349984:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829908.349988:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829908.349995:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391232:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829908.349998:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829908.350000:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350004:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829908.350005:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.350006:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829908.350008:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829908.350009:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829908.350011:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391360:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829908.350012:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829908.350014:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829908.350015:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.350016:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829908.350018:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829908.350019:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.350022:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829908.350023:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829908.350025:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829908.350027:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391360, portal 4
00000100:00000001:0.0:1787829908.350028:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829908.350029:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:0.0:1787829908.350032:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350038:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829908.350040:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829908.350042:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829908.350044:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327391360, offset 0
00000400:00000200:0.0:1787829908.350048:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829908.350052:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.350056:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829908.350058:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829908.350061:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829908.350063:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829908.350065:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829908.350068:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40829:40829:256:4294967295] 192.168.203.116@tcp LPNI seq info [40829:40829:8:4294967295]
00000400:00000200:0.0:1787829908.350072:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.350090:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829908.350094:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829908.350098:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829908.350101:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829908.350104:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829908.350108:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829908.350111:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829908.350114:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829908.350116:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829908.350119:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829908.350122:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829908.350125:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829908.350127:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829908.350129:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829908.350133:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350140:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.350143:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350149:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.350151:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829908.350154:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829908.350156:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829908.350159:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829908.350161:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298728466
00000100:00000001:0.0:1787829908.350164:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298729132
00000100:00000001:0.0:1787829908.350165:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298728466
00000100:00000001:0.0:1787829908.350167:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298729132
00000100:00000001:0.0:1787829908.350170:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829908.350172:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.350173:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829908.350177:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829908/real 1787829908]  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829908.350185:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829908.350186:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829908.350188:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829908.350190:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829908.350192:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829908.350194:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829908.350197:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829908.350203:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350206:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829908.350207:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350208:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.350209:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829908.350211:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350215:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829908.350217:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350221:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829908.350222:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350223:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.350224:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350225:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829908.350227:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829908.350228:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829908.350230:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829908.350231:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829908.350233:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829908.350237:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391360:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829908.350240:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829908.350241:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350245:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829908.350246:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.350247:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829908.350248:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829908.350249:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829908.350251:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391488:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829908.350253:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829908.350254:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829908.350255:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829908.350256:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829908.350258:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829908.350259:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829908.350262:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829908.350263:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829908.350265:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829908.350267:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391488, portal 4
00000100:00000001:0.0:1787829908.350268:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829908.350268:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:0.0:1787829908.350272:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350276:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829908.350277:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829908.350278:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829908.350280:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327391488, offset 0
00000400:00000200:0.0:1787829908.350282:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829908.350284:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.350286:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829908.350288:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829908.350289:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829908.350291:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829908.350293:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829908.350294:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40830:40830:256:4294967295] 192.168.203.116@tcp LPNI seq info [40830:40830:8:4294967295]
00000400:00000200:0.0:1787829908.350297:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829908.350301:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829908.350303:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829908.350305:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829908.350307:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829908.350309:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787829908.350311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829908.350313:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1787829908.350315:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829908.350316:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829908.350318:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829908.350320:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829908.350322:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829908.350323:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829908.350325:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829908.350326:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350331:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.350332:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350340:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.350341:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829908.350342:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829908.350344:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829908.350346:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829908.350347:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298728471
00000100:00000001:0.0:1787829908.350348:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298729132
00000100:00000001:0.0:1787829908.350350:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298728471
00000100:00000001:0.0:1787829908.350351:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298729132
00000100:00000001:0.0:1787829908.350352:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829908.350353:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.350354:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829908.350356:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829908/real 1787829908]  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829908.350360:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829908.350361:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829908.350361:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829908.350363:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829908.350364:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829908.350365:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829908.350366:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829908.350370:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350374:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829908.350375:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350376:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.350377:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829908.350378:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350382:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829908.350385:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350388:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829908.350389:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350390:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829908.350391:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350392:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829908.350393:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829908.350394:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829908.350395:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829908.350397:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829908.350398:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829908.350402:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391488:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829908.350404:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350405:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.350407:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327391232/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350410:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829908.350411:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829908.350412:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829908.350413:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829908.350414:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829908.350415:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829908.350416:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829908.350417:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829908.350419:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829908.350420:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829908.350421:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:0.0:1787829908.350423:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829908.350424:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350425:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.350426:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327391360/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350430:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829908.350431:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829908.350431:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829908.350433:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829908.350433:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829908.350434:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829908.350435:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829908.350436:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:0.0:1787829908.350436:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829908.350437:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829908.350438:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:0.0:1787829908.350439:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829908.350439:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350440:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829908.350442:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327391488/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829908.350445:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829908.350445:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829908.350446:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829908.350447:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829908.350448:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829908.350448:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829908.350449:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829908.350450:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829908.350451:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829908.350451:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829908.350452:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:0.0:1787829908.350453:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829908.350454:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350455:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.350457:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829908.350458:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.350459:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829908.350460:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829908.860720:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829908.860725:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829908.860727:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829908.860730:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829908.860733:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829908.860736:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.860738:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829908.860739:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829908.860741:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829909.052727:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298729132, at 4298729152
00000100:00000040:3.0:1787829909.052727:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298729132, at 4298729152
00000100:00000010:2.0:1787829909.052739:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc6c0.
00000100:00000010:3.0:1787829909.052742:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccea0.
00010000:00000001:0.0:1787829909.180752:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829909.180758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829909.180764:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829909.180766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829909.180769:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829909.180770:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829909.180817:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829909.180820:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829909.180822:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829909.884930:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829909.885086:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829909.885089:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829909.885094:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829909.885097:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829909.885100:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829909.885102:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829909.885105:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829909.885108:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829910.205363:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829910.205370:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829910.205375:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829910.205376:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829910.205379:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829910.205382:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829910.205384:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829910.205387:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829910.205389:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829910.205392:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829910.205393:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829910.205395:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829910.205466:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829910.205473:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829910.205477:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829910.908776:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829910.908783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829910.908785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829910.908791:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829910.908794:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829910.908798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829910.908800:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829910.908803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829910.908806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829911.228795:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829911.228804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829911.228807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829911.228808:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829911.228812:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829911.228813:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829911.228845:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829911.228848:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829911.228850:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829911.932785:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829911.932794:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829911.932797:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829911.932802:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829911.932807:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829911.932811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829911.932814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829911.932816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829911.932819:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829912.252772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829912.252779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829912.252785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829912.252787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829912.252790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829912.252792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829912.252844:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829912.252848:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829912.252852:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829912.956794:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829912.956801:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829912.956804:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829912.956808:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829912.956812:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829912.956815:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829912.956817:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829912.956818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829912.956822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829913.276784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829913.276791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829913.276799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829913.276800:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829913.276803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829913.276805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829913.276877:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829913.276881:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829913.276884:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829913.468817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829913.468828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829913.468830:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829913.468833:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829913.468837:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829913.468844:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4061
00000400:00000200:1.0:1787829913.468853:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829913.468860:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829913.468864:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829913.468867:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.468870:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829913.468873:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.468875:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829913.468878:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829913.468880:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829913.468889:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829913.468892:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829913.468894:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829913.468895:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829913.468897:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.468899:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829913.468900:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.468901:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829913.468904:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829913.468906:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829913.468907:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829913.468909:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829913.468911:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829913.468913:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.468915:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829913.468916:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829913.468924:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000020:00000040:1.0:1787829913.468926:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829913.468929:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829913.468930:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.468932:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829913.468934:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829913.468937:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829913.468942:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000100:00000001:1.0:1787829913.468946:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829913.468953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829913.468960:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829913.468962:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829913.468965:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829913.468982:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829913.468984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829913.468986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829913.468987:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829913.468988:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829913.468990:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829913.468991:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4061
00000400:00000200:1.0:1787829913.468994:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829913.468995:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829913.468997:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829913.468999:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469001:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829913.469002:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469004:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829913.469005:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829913.469006:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829913.469008:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829913.469010:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829913.469011:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829913.469013:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829913.469016:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1787829913.469016:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469017:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829913.469021:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829913.469025:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829913.469026:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469031:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829913.469033:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:0.0:1787829913.469037:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829913.469037:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469053:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829913.469056:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829913.469061:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829913.469065:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829913.469065:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469066:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829913.469070:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787829913.469071:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829913.469076:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469078:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787829913.469078:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829913.469081:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
02000000:00000001:0.0:1787829913.469083:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829913.469084:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829913.469089:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829913.469090:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829913.469090:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469096:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829913.469097:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829913.469099:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829913.469101:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327391616:192.168.203.116@tcp:250:kworker.0
02000000:00000010:1.0:1787829913.469102:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:1.0:1787829913.469108:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469109:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829913.469114:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:1.0:1787829913.469115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829913.469116:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469121:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787829913.469126:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000010:0.0:1787829913.469127:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000100:00000001:1.0:1787829913.469128:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469132:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829913.469134:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829913.469141:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829913.469146:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00000001:1.0:1787829913.469147:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829913.469153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787829913.469155:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00080000:1.0:1787829913.469157:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829913.469159:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829913.469161:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1787829913.469162:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391616, portal 25
00000100:00000001:1.0:1787829913.469166:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829913.469167:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829913.469170:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00080000:1.0:1787829913.469173:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4061
00000100:00000040:0.0:1787829913.469178:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829913.469183:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829913.469191:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829913.469192:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1787829913.469196:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829913.469197:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829913.469201:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000040:1.0:1787829913.469203:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829913.469205:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327391616, offset 0
00000100:00000001:1.0:1787829913.469206:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829913.469209:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469211:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829913.469215:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000001:1.0:1787829913.469215:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829913.469218:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829913.469219:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829913.469221:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829913.469222:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829913.469226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:1.0:1787829913.469226:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829913.469231:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469234:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829913.469235:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:1.0:1787829913.469238:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469241:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829913.469242:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1787829913.469243:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829913.469248:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:1.0:1787829913.469248:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469252:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829913.469254:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787829913.469255:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829913.469259:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
02000000:00000001:1.0:1787829913.469259:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829913.469263:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469267:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829913.469268:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40831:40831:256:4294967295] 192.168.203.116@tcp LPNI seq info [40831:40831:8:4294967295]
00000008:00100000:1.0:1787829913.469271:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829913.469275:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00000001:1.0:1787829913.469275:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469278:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829913.469281:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829913.469285:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829913.469286:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1787829913.469289:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829913.469292:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469295:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.0:1787829913.469296:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:1.0:1787829913.469297:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829913.469299:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829913.469301:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
02000000:00000010:1.0:1787829913.469303:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829913.469307:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829913.469308:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829913.469313:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00100000:1.0:1787829913.469314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829913.469323:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829913.469327:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:0.1:1787829913.469328:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000100:00000001:1.0:1787829913.469330:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829913.469337:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000040:1.0:1787829913.469338:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1787829913.469340:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000100:00000001:1.0:1787829913.469347:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829913.469348:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1787829913.469349:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829913.469352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829913.469355:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829913.469356:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1787829913.469357:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829913.469360:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829913.469361:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00080000:1.0:1787829913.469363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4061
00000400:00000010:0.0:1787829913.469364:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829913.469369:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:1.0:1787829913.469370:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829913.469374:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829913.469378:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829913.469383:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829913.469388:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829913.469389:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829913.469393:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829913.469394:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829913.469400:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829913.469402:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469404:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829913.469405:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829913.469410:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829913.469410:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829913.469413:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000200:0.0:1787829913.469417:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000001:1.0:1787829913.469417:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829913.469420:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298734252
00000020:00000040:1.0:1787829913.469421:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000010:0.0:1787829913.469426:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc780.
00000020:00000001:1.0:1787829913.469427:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829913.469432:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829913.469435:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787829913.469435:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469440:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829913.469441:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469443:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829913.469444:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829913.469445:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298734252
00000100:00080000:1.0:1787829913.469447:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829913.469451:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829913.469452:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc420.
02000000:00000001:1.0:1787829913.469452:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829913.469456:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829913.469458:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469459:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829913.469460:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829913.469462:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829913.469464:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469465:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829913.469474:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829913.469475:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829913.469477:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829913.469477:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829913.469479:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829913.469483:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829913.469484:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829913.469485:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829913.469487:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469489:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:1.0:1787829913.469491:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00100000:0.0:1787829913.469496:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913]  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829913.469497:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829913.469504:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829913.469521:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:0.0:1787829913.469522:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787829913.469523:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829913.469527:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:1.0:1787829913.469528:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787829913.469531:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829913.469537:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000100:00000001:1.0:1787829913.469537:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829913.469540:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000040:1.0:1787829913.469541:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4071)
00000100:00000001:0.0:1787829913.469543:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829913.469549:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829913.469554:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469558:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829913.469560:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469561:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.469563:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829913.469566:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469570:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829913.469574:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469591:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829913.469594:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469596:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.469597:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469599:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829913.469601:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829913.469603:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829913.469607:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829913.469610:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829913.469614:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829913.469622:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327391616:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829913.469627:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469629:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.469631:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327391616/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469637:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829913.469639:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829913.469641:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829913.469644:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829913.469645:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829913.469647:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829913.469649:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829913.469651:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829913.469654:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829913.469691:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829913.469693:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829913.469696:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829913.469697:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469698:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469704:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829913.469705:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469706:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829913.469707:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829913.469708:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829913.469711:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469715:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829913.469716:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469718:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829913.469719:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829913.469720:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829913.469722:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391744:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829913.469724:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829913.469725:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829913.469726:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469727:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829913.469729:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829913.469731:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829913.469733:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829913.469735:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829913.469737:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829913.469739:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391744, portal 10
00000100:00000001:0.0:1787829913.469740:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829913.469741:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829913.469746:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469750:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829913.469751:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829913.469753:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829913.469754:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327391744, offset 0
00000400:00000200:0.0:1787829913.469757:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829913.469759:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829913.469762:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829913.469764:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829913.469766:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829913.469767:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829913.469769:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829913.469771:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40832:40832:256:4294967295] 192.168.203.116@tcp LPNI seq info [40832:40832:8:4294967295]
00000400:00000200:0.0:1787829913.469773:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829913.469777:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829913.469779:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829913.469781:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829913.469783:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829913.469785:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829913.469788:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829913.469789:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829913.469791:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829913.469792:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829913.469794:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829913.469796:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829913.469798:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829913.469800:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829913.469801:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829913.469803:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469808:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.469810:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469814:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469815:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829913.469817:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829913.469819:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829913.469821:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829913.469822:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298733579
00000100:00000001:0.0:1787829913.469824:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298734252
00000100:00000001:0.0:1787829913.469826:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298733579
00000100:00000001:0.0:1787829913.469827:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298734252
00000100:00000001:0.0:1787829913.469828:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829913.469830:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469831:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829913.469833:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913]  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829913.469837:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829913.469837:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829913.469838:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829913.469839:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829913.469840:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829913.469842:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829913.469844:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829913.469849:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469852:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829913.469853:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469854:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.469854:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829913.469856:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469860:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829913.469862:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469866:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829913.469867:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469868:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.469869:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.469870:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829913.469871:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829913.469873:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829913.469875:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829913.469876:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829913.469878:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829913.469882:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391744:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829913.469884:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829913.469885:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469889:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829913.469890:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469891:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829913.469892:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829913.469893:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829913.469895:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391872:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829913.469896:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829913.469897:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829913.469898:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.469899:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829913.469901:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829913.469902:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829913.469904:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829913.469905:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829913.469907:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829913.469909:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327391872, portal 4
00000100:00000001:0.0:1787829913.469910:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829913.469910:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829913.469914:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469917:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829913.469918:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829913.469920:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829913.469921:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327391872, offset 0
00000400:00000200:0.0:1787829913.469923:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829913.469925:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829913.469926:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829913.469928:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829913.469930:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829913.469931:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829913.469932:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829913.469934:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40833:40833:256:4294967295] 192.168.203.116@tcp LPNI seq info [40833:40833:8:4294967295]
00000400:00000200:0.0:1787829913.469936:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829913.469939:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829913.469941:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829913.469943:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829913.469945:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829913.469946:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829913.469948:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829913.469951:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829913.469953:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829913.469954:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829913.469955:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829913.469957:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829913.469959:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829913.469960:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829913.469961:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829913.469964:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469971:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.469974:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.469980:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.469981:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829913.469984:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829913.469986:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829913.469989:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829913.469991:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298733586
00000100:00000001:0.0:1787829913.469994:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298734252
00000100:00000001:0.0:1787829913.469995:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298733586
00000100:00000001:0.0:1787829913.469997:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298734252
00000100:00000001:0.0:1787829913.469999:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829913.470001:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.470003:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829913.470007:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913]  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829913.470014:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829913.470015:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829913.470016:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829913.470019:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829913.470020:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829913.470022:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829913.470025:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829913.470033:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470058:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829913.470059:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470061:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.470062:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829913.470065:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470069:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829913.470071:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470075:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829913.470076:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470077:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.470078:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470079:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829913.470080:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829913.470081:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829913.470083:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829913.470084:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829913.470086:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829913.470090:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327391872:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829913.470093:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829913.470094:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470098:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829913.470099:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.470100:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829913.470101:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829913.470103:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829913.470104:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392000:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829913.470106:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829913.470107:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829913.470108:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829913.470109:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829913.470111:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829913.470112:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829913.470115:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829913.470116:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829913.470118:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829913.470120:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392000, portal 4
00000100:00000001:0.0:1787829913.470121:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829913.470122:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00000040:0.0:1787829913.470126:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470129:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829913.470131:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829913.470132:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829913.470133:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327392000, offset 0
00000400:00000200:0.0:1787829913.470136:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829913.470138:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829913.470140:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829913.470142:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829913.470143:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829913.470145:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829913.470146:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829913.470148:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40834:40834:256:4294967295] 192.168.203.116@tcp LPNI seq info [40834:40834:8:4294967295]
00000400:00000200:0.0:1787829913.470150:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829913.470154:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829913.470157:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829913.470159:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829913.470161:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829913.470163:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787829913.470165:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829913.470167:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787829913.470169:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829913.470170:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829913.470173:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829913.470174:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829913.470176:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829913.470177:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829913.470179:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829913.470181:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470185:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.470187:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470191:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.470192:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829913.470194:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829913.470195:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829913.470197:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829913.470198:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298733591
00000100:00000001:0.0:1787829913.470200:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298734252
00000100:00000001:0.0:1787829913.470202:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298733591
00000100:00000001:0.0:1787829913.470203:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298734252
00000100:00000001:0.0:1787829913.470204:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829913.470205:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.470206:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829913.470208:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913]  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829913.470213:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829913.470213:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829913.470214:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829913.470216:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829913.470216:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829913.470217:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829913.470219:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829913.470223:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470226:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829913.470227:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470228:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.470229:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829913.470231:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470235:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829913.470237:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470240:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829913.470241:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470242:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829913.470243:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470244:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829913.470245:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829913.470246:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829913.470247:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829913.470249:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829913.470251:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829913.470258:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392000:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829913.470261:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470263:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.470265:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327391744/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470272:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829913.470273:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829913.470275:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829913.470277:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829913.470278:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829913.470280:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829913.470282:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829913.470283:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829913.470285:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829913.470287:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829913.470288:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829913.470290:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829913.470292:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470293:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.470296:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327391872/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470302:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829913.470303:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829913.470304:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829913.470305:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829913.470306:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829913.470307:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829913.470308:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829913.470309:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829913.470310:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829913.470311:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829913.470312:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829913.470313:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829913.470314:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470315:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829913.470317:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327392000/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829913.470320:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829913.470321:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829913.470322:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829913.470324:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829913.470325:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829913.470326:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829913.470327:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829913.470329:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829913.470330:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829913.470331:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829913.470333:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829913.470335:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829913.470336:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470338:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.470341:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829913.470342:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.470344:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829913.470346:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829913.980777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829913.980783:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829913.980785:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829913.980790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829913.980793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829913.980796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.980798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829913.980801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829913.980803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829914.172799:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298734252, at 4298734272
00000100:00000040:2.0:1787829914.172803:0:16289:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298734252, at 4298734272
00000100:00000010:3.0:1787829914.172818:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc420.
00000100:00000010:2.0:1787829914.172822:0:16289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc780.
00010000:00000001:0.0:1787829914.300787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829914.300797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829914.300803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829914.300805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829914.300808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829914.300809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829914.300839:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829914.300842:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829914.300845:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829915.004778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829915.004784:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829915.004786:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829915.004792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829915.004795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829915.004798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829915.004801:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829915.004803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829915.004806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829915.324765:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829915.324768:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829915.324771:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324776:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829915.324778:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324781:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324797:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324805:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829915.324807:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829915.324809:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829915.324810:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324812:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829915.324813:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324814:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324816:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829915.324819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829915.324820:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829915.324884:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829915.324892:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829915.324896:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829916.028773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829916.028781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829916.028783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829916.028788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829916.028792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829916.028796:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829916.028799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829916.028802:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829916.028805:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829916.348803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829916.348812:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829916.348818:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829916.348819:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829916.348823:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829916.348824:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829916.348894:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829916.348900:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829916.348905:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829917.052761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829917.052767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829917.052769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829917.052774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829917.052777:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829917.052781:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829917.052783:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829917.052786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829917.052788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829917.372745:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829917.372750:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829917.372754:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829917.372755:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829917.372757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829917.372758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829917.372772:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829917.372774:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829917.372776:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829918.076775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829918.076782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829918.076783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829918.076787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829918.076791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829918.076794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.076796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829918.076798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.076800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829918.396802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829918.396818:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829918.396832:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829918.396834:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829918.396840:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829918.396842:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829918.396916:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829918.396922:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829918.396927:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829918.588770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829918.588784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829918.588787:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829918.588791:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829918.588796:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829918.588806:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4066
00000400:00000200:1.0:1787829918.588819:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829918.588828:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829918.588832:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829918.588836:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.588839:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.588842:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.588844:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829918.588849:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829918.588851:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829918.588854:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829918.588858:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829918.588861:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829918.588863:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829918.588866:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.588868:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.588870:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.588872:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829918.588876:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829918.588879:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829918.588882:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829918.588885:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829918.588889:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829918.588892:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.588894:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829918.588897:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829918.588910:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2940.
00000020:00000040:1.0:1787829918.588914:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829918.588918:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829918.588920:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.588924:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829918.588927:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829918.588933:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829918.588944:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829918.588952:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829918.588964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829918.588975:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829918.588981:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829918.588985:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bf16e7d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829918.589028:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829918.589032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829918.589035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829918.589037:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829918.589038:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829918.589041:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829918.589044:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4066
00000400:00000200:1.0:1787829918.589049:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829918.589052:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829918.589055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829918.589065:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589067:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.589070:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589072:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829918.589076:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829918.589077:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829918.589079:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829918.589081:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829918.589083:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829918.589085:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829918.589089:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589091:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.589093:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589095:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829918.589097:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829918.589101:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829918.589102:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829918.589104:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829918.589112:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829918.589113:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829918.589118:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1787829918.589118:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829918.589124:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589126:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829918.589131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000040:0.0:1787829918.589135:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829918.589136:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589142:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829918.589147:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3480.
00000020:00000040:1.0:1787829918.589150:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829918.589157:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829918.589160:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589166:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829918.589171:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829918.589173:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829918.589175:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829918.589177:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.589181:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:1.0:1787829918.589184:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829918.589188:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829918.589190:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.589194:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829918.589200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829918.589202:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327392128:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829918.589217:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829918.589223:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829918.589224:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829918.589228:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.589230:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.589235:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1787829918.589237:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000997132d4] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829918.589246:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829918.589250:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589255:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829918.589260:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829918.589261:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:1.0:1787829918.589269:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:0.0:1787829918.589270:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00000001:1.0:1787829918.589272:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829918.589274:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829918.589276:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829918.589283:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4066
00000400:00000010:0.0:1787829918.589286:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000400:00000200:1.0:1787829918.589291:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829918.589298:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392128, portal 25
00000100:00000001:0.0:1787829918.589304:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000400:00000200:1.0:1787829918.589304:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829918.589308:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419004224 : -116696290547392 : ffff95dd870e2940)
00000100:00080000:1.0:1787829918.589310:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829918.589316:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829918.589321:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829918.589321:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.589329:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589334:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829918.589340:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829918.589342:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000001:0.0:1787829918.589348:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:1.0:1787829918.589353:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829918.589358:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000100:00000040:0.0:1787829918.589359:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000020:00000001:1.0:1787829918.589364:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:0.0:1787829918.589367:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000020:00000001:1.0:1787829918.589370:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000200:0.0:1787829918.589374:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327392128, offset 0
00000100:00000040:1.0:1787829918.589378:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589383:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.589388:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829918.589391:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589394:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829918.589398:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829918.589401:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829918.589403:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829918.589411:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829918.589414:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829918.589417:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829918.589420:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589422:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829918.589429:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000008:00100000:1.0:1787829918.589430:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829918.589436:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829918.589438:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829918.589440:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829918.589446:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3840.
00000400:00000200:0.0:1787829918.589447:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000040:1.0:1787829918.589453:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829918.589456:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787829918.589460:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829918.589464:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829918.589465:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589472:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829918.589476:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829918.589480:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40835:40835:256:4294967295] 192.168.203.116@tcp LPNI seq info [40835:40835:8:4294967295]
02000000:00000001:1.0:1787829918.589481:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829918.589490:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000400:00000200:0.0:1787829918.589496:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829918.589498:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829918.589508:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829918.589515:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829918.589531:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589536:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829918.589540:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829918.589542:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00000040:1.0:1787829918.589549:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076ec8bdb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829918.589559:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829918.589564:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829918.589577:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:1.0:1787829918.589581:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829918.589583:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000100:00080000:1.0:1787829918.589588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:0.0:1787829918.589597:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829918.589599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000010:0.0:1787829918.589602:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000100:00000001:1.0:1787829918.589603:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829918.589607:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829918.589612:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829918.589613:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1787829918.589620:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4066
00000400:00000200:0.0:1787829918.589627:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829918.589634:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.589639:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000200:1.0:1787829918.589641:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000010:0.0:1787829918.589646:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00080000:1.0:1787829918.589648:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829918.589653:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787829918.589680:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589683:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.589685:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589688:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829918.589692:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829918.589693:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829918.589695:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829918.589698:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829918.589701:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829918.589705:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000200:0.0:1787829918.589710:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829918.589712:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589721:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829918.589728:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589735:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829918.589741:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829918.589742:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:1.0:1787829918.589745:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829918.589751:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829918.589752:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829918.589755:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829918.589761:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829918.589769:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.589771:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589774:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829918.589776:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000008:00100000:1.0:1787829918.589779:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829918.589784:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.589785:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829918.589788:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829918.589795:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:1.0:1787829918.589796:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829918.589799:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298739373
00000020:00000040:1.0:1787829918.589801:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829918.589808:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829918.589809:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829918.589810:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc8a0.
00000100:00000001:1.0:1787829918.589812:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829918.589816:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829918.589818:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.589821:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
02000000:00000010:1.0:1787829918.589824:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:0.0:1787829918.589829:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298739373
00000100:00000001:1.0:1787829918.589830:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829918.589839:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccb40.
00000100:00100000:1.0:1787829918.589842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829918.589846:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829918.589858:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829918.589861:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.589867:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787829918.589867:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d08b10bb] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.589877:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.589887:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787829918.589891:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829918.589899:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829918]  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829918.589902:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4076)
00000400:00000001:0.0:1787829918.589916:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829918.589918:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829918.589921:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829918.589926:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829918.589927:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829918.589931:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829918.589935:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829918.589943:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.589949:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829918.589951:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.589953:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.589955:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829918.589958:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.589965:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829918.589969:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.589976:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829918.589980:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.589981:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.589983:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.589985:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829918.589987:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829918.589991:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829918.589994:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829918.589997:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829918.590001:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829918.590021:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327392128:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829918.590026:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590028:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.590031:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2940 x1874671327392128/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590037:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829918.590040:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829918.590043:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829918.590046:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829918.590049:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829918.590051:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829918.590053:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829918.590055:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829918.590059:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829918.590061:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829918.590062:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2940.
00000100:00000001:0.0:1787829918.590066:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829918.590067:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590069:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590076:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829918.590078:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590080:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829918.590082:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829918.590083:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829918.590087:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590094:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829918.590095:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.590098:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829918.590100:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829918.590102:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829918.590104:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392256:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829918.590107:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829918.590110:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829918.590111:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.590113:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829918.590116:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829918.590118:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829918.590121:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829918.590123:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829918.590126:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829918.590129:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392256, portal 10
00000100:00000001:0.0:1787829918.590131:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829918.590133:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419007104 : -116696290544512 : ffff95dd870e3480)
00000100:00000040:0.0:1787829918.590139:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590145:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829918.590147:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829918.590149:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829918.590151:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327392256, offset 0
00000400:00000200:0.0:1787829918.590155:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829918.590159:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.590162:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829918.590165:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829918.590168:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829918.590171:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829918.590173:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829918.590176:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40836:40836:256:4294967295] 192.168.203.116@tcp LPNI seq info [40836:40836:8:4294967295]
00000400:00000200:0.0:1787829918.590180:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.590186:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829918.590190:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829918.590194:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829918.590197:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829918.590200:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829918.590204:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829918.590207:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829918.590211:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829918.590212:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829918.590215:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829918.590218:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829918.590222:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829918.590223:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829918.590226:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829918.590229:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590237:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.590240:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590246:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590248:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829918.590251:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829918.590255:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829918.590258:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829918.590260:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298738699
00000100:00000001:0.0:1787829918.590263:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298739373
00000100:00000001:0.0:1787829918.590265:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298738699
00000100:00000001:0.0:1787829918.590268:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298739373
00000100:00000001:0.0:1787829918.590270:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829918.590272:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590274:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829918.590278:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829918]  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829918.590286:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829918.590287:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829918.590289:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829918.590291:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829918.590293:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829918.590295:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829918.590298:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829918.590306:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590312:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829918.590313:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590315:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.590316:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829918.590319:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590326:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829918.590330:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590337:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829918.590339:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590341:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.590342:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590344:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829918.590346:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829918.590349:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829918.590352:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829918.590354:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829918.590358:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829918.590367:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392256:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829918.590370:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829918.590374:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590381:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829918.590382:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.590384:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829918.590386:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829918.590388:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829918.590391:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392384:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829918.590394:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829918.590397:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829918.590398:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.590400:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829918.590403:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829918.590405:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829918.590408:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829918.590410:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829918.590413:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829918.590416:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392384, portal 4
00000100:00000001:0.0:1787829918.590418:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829918.590419:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419008064 : -116696290543552 : ffff95dd870e3840)
00000100:00000040:0.0:1787829918.590425:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590431:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829918.590434:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829918.590436:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829918.590439:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327392384, offset 0
00000400:00000200:0.0:1787829918.590443:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829918.590447:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.590450:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829918.590452:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829918.590455:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829918.590457:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829918.590459:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829918.590463:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40837:40837:256:4294967295] 192.168.203.116@tcp LPNI seq info [40837:40837:8:4294967295]
00000400:00000200:0.0:1787829918.590466:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.590472:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829918.590477:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829918.590480:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829918.590484:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829918.590486:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829918.590490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829918.590493:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829918.590496:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829918.590498:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829918.590501:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829918.590504:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829918.590507:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829918.590508:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829918.590511:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829918.590514:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590521:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.590524:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590536:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590538:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829918.590541:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829918.590543:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829918.590547:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829918.590548:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298738706
00000100:00000001:0.0:1787829918.590551:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298739373
00000100:00000001:0.0:1787829918.590553:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298738706
00000100:00000001:0.0:1787829918.590554:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298739373
00000100:00000001:0.0:1787829918.590557:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829918.590559:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590561:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829918.590565:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829918]  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829918.590571:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829918.590573:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829918.590574:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829918.590577:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829918.590578:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829918.590580:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829918.590583:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829918.590591:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590597:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829918.590599:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590600:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.590602:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829918.590605:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590612:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829918.590615:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590622:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829918.590624:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590625:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.590627:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590628:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829918.590630:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829918.590632:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829918.590635:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829918.590637:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829918.590640:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829918.590647:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392384:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829918.590651:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829918.590675:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590683:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829918.590684:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.590686:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829918.590688:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829918.590690:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829918.590693:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392512:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829918.590696:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829918.590697:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829918.590699:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829918.590701:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829918.590704:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829918.590706:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829918.590709:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829918.590711:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829918.590714:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829918.590717:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392512, portal 4
00000100:00000001:0.0:1787829918.590718:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829918.590720:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418995584 : -116696290556032 : ffff95dd870e0780)
00000100:00000040:0.0:1787829918.590725:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590731:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829918.590733:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829918.590735:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829918.590737:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327392512, offset 0
00000400:00000200:0.0:1787829918.590741:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829918.590745:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.590748:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829918.590751:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829918.590753:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829918.590755:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829918.590757:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829918.590760:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40838:40838:256:4294967295] 192.168.203.116@tcp LPNI seq info [40838:40838:8:4294967295]
00000400:00000200:0.0:1787829918.590764:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829918.590770:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829918.590774:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829918.590777:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829918.590780:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829918.590783:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787829918.590787:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829918.590789:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787829918.590793:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829918.590795:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829918.590798:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829918.590801:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829918.590804:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829918.590805:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829918.590807:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829918.590811:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590819:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.590821:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590827:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590829:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829918.590832:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829918.590834:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829918.590838:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829918.590840:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298738712
00000100:00000001:0.0:1787829918.590842:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298739373
00000100:00000001:0.0:1787829918.590843:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298738712
00000100:00000001:0.0:1787829918.590845:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298739373
00000100:00000001:0.0:1787829918.590847:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829918.590849:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.590851:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829918.590854:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829918]  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829918.590861:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829918.590862:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829918.590864:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829918.590866:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829918.590867:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829918.590869:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829918.590873:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829918.590880:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590886:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829918.590888:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590890:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.590891:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829918.590894:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590900:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829918.590903:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590910:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829918.590912:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590913:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829918.590915:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590917:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829918.590918:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829918.590920:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829918.590924:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829918.590926:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829918.590930:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829918.590937:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392512:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829918.590941:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590943:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.590946:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3480 x1874671327392256/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590952:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829918.590954:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829918.590956:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829918.590958:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829918.590960:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829918.590961:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829918.590963:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829918.590965:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829918.590967:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829918.590968:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829918.590970:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3480.
00000100:00000001:0.0:1787829918.590972:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829918.590973:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.590975:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.590978:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3840 x1874671327392384/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.590983:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829918.590984:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829918.590986:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829918.590988:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829918.590989:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829918.590990:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829918.590991:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829918.590993:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829918.590995:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829918.590996:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829918.590998:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3840.
00000100:00000001:0.0:1787829918.591000:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829918.591001:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.591003:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829918.591019:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0780 x1874671327392512/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829918.591026:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829918.591028:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829918.591029:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829918.591031:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829918.591033:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829918.591034:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829918.591036:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829918.591037:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829918.591039:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829918.591040:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829918.591041:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0780.
00000100:00000001:0.0:1787829918.591044:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829918.591045:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.591046:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829918.591049:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829918.591051:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829918.591052:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829918.591054:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829919.100789:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829919.100796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829919.100798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829919.100804:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829919.100808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829919.100812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829919.100815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829919.100817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829919.100820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829919.292755:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298739373, at 4298739392
00000100:00000040:3.0:1787829919.292755:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298739373, at 4298739392
00000100:00000010:2.0:1787829919.292768:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc8a0.
00000100:00000010:3.0:1787829919.292770:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccb40.
00010000:00000001:0.0:1787829919.420750:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829919.420757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829919.420762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829919.420763:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829919.420766:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829919.420767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829919.420818:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829919.420821:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829919.420823:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829920.124766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829920.124773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829920.124775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829920.124780:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829920.124784:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829920.124787:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829920.124790:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829920.124793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829920.124796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829920.444761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829920.444767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829920.444772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829920.444773:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829920.444776:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829920.444779:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829920.444781:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829920.444784:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829920.444787:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829920.444789:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829920.444791:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829920.444793:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829920.444812:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829920.444815:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829920.444817:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829921.148757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829921.148762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829921.148763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829921.148766:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829921.148769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829921.148771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829921.148773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829921.148775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829921.148777:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829921.468741:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829921.468757:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829921.468763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829921.468764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829921.468767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829921.468769:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829921.468812:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829921.468815:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829921.468817:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829922.172761:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829922.172767:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829922.172769:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829922.172774:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829922.172778:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829922.172782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829922.172785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829922.172787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829922.172790:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829922.492749:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829922.492753:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829922.492757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829922.492758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829922.492761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829922.492761:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829922.492809:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829922.492812:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829922.492814:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829923.196749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829923.196755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829923.196756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829923.196759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829923.196762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829923.196764:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.196766:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829923.196767:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.196770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829923.516733:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829923.516738:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829923.516742:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829923.516743:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829923.516745:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829923.516746:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829923.516760:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829923.516762:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829923.516764:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829923.708747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829923.708754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829923.708756:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829923.708758:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829923.708762:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829923.708767:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4071
00000400:00000200:1.0:1787829923.708774:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829923.708779:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829923.708780:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829923.708783:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708784:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.708787:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708788:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829923.708791:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829923.708792:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829923.708801:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829923.708803:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829923.708804:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829923.708806:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829923.708808:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708809:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.708810:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708811:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829923.708813:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829923.708815:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708816:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829923.708818:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829923.708820:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829923.708822:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.708823:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829923.708825:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829923.708833:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829923.708835:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829923.708838:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829923.708839:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.708841:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829923.708843:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829923.708846:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829923.708851:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4c00.
00000100:00000001:1.0:1787829923.708856:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829923.708862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829923.708870:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829923.708873:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829923.708876:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829923.708891:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829923.708893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829923.708895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829923.708895:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829923.708896:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829923.708898:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829923.708900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4071
00000400:00000200:1.0:1787829923.708902:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829923.708904:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829923.708906:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829923.708909:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708910:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.708912:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708913:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829923.708915:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787829923.708915:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829923.708917:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829923.708919:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829923.708920:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1787829923.708921:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000100:00000001:0.0:1787829923.708924:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787829923.708924:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829923.708926:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829923.708931:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829923.708933:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829923.708933:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.708935:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708938:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829923.708942:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829923.708947:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829923.708953:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829923.708954:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829923.708957:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829923.708959:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787829923.708960:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.708963:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.708963:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1787829923.708969:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:1.0:1787829923.708969:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787829923.708981:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1787829923.708982:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.708984:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.708987:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829923.708988:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000100:00100000:0.0:1787829923.708992:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327392640:192.168.203.116@tcp:250:kworker.0
00000020:00000040:1.0:1787829923.708993:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829923.708997:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829923.708999:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829923.708999:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709002:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829923.709003:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829923.709003:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829923.709004:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709006:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829923.709008:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4800.
02000000:00000001:0.0:1787829923.709009:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829923.709016:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000100:00000001:1.0:1787829923.709017:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709019:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829923.709023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829923.709027:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829923.709031:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00000001:1.0:1787829923.709034:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829923.709036:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829923.709038:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000040:1.0:1787829923.709040:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829923.709046:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392640, portal 25
00000100:00000001:1.0:1787829923.709047:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709049:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1787829923.709050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829923.709052:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00080000:1.0:1787829923.709055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829923.709058:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1787829923.709061:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829923.709061:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829923.709063:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829923.709068:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4071
00000400:00000200:1.0:1787829923.709075:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829923.709076:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787829923.709079:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000040:0.0:1787829923.709084:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829923.709084:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000010:0.0:1787829923.709088:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000040:1.0:1787829923.709090:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829923.709092:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327392640, offset 0
00000100:00000001:1.0:1787829923.709093:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.709097:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709099:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829923.709101:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829923.709102:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:1.0:1787829923.709103:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829923.709106:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829923.709107:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829923.709108:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829923.709110:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829923.709113:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829923.709116:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709118:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.709121:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709122:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829923.709123:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829923.709127:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:0.0:1787829923.709128:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1787829923.709131:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709134:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709135:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:1.0:1787829923.709135:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1787829923.709139:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787829923.709139:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829923.709144:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709145:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709147:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829923.709150:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829923.709151:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40839:40839:256:4294967295] 192.168.203.116@tcp LPNI seq info [40839:40839:8:4294967295]
00000008:00000001:1.0:1787829923.709153:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709156:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829923.709159:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829923.709161:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829923.709162:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:1.0:1787829923.709162:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829923.709169:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709171:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829923.709172:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829923.709175:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709179:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000010:1.0:1787829923.709181:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4400.
00000100:00000001:1.0:1787829923.709184:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829923.709186:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00100000:1.0:1787829923.709190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1787829923.709193:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00000001:1.0:1787829923.709199:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829923.709200:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829923.709203:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829923.709205:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829923.709208:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787829923.709215:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829923.709219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1787829923.709219:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:1.0:1787829923.709222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829923.709223:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829923.709224:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829923.709225:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829923.709227:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4071
00000400:00000200:1.0:1787829923.709229:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829923.709231:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829923.709234:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829923.709235:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709236:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829923.709237:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709241:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829923.709242:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.1:1787829923.709242:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000020:00000040:1.0:1787829923.709244:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829923.709247:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829923.709248:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000800:00000100:0.0:1787829923.709250:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000020:00000001:1.0:1787829923.709251:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000800:00000010:0.0:1787829923.709254:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000020:00000001:1.0:1787829923.709254:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829923.709260:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709261:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829923.709262:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1787829923.709264:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829923.709266:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829923.709270:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000200:0.0:1787829923.709271:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1787829923.709275:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709277:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00000001:1.0:1787829923.709278:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829923.709282:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
02000000:00000001:1.0:1787829923.709282:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829923.709285:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829923.709287:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709288:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787829923.709289:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829923.709291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000200:0.0:1787829923.709295:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787829923.709295:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709298:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829923.709302:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829923.709305:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829923.709308:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829923.709309:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1787829923.709310:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709312:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1787829923.709315:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829923.709315:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829923.709318:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829923.709323:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7c00.
00000100:00000001:0.0:1787829923.709326:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709327:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709331:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:1.0:1787829923.709335:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829923.709337:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829923.709341:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829923.709344:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709345:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829923.709348:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709349:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298744492
00000100:00000010:0.0:1787829923.709356:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc540.
00000100:00000001:1.0:1787829923.709357:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829923.709362:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4081)
00000100:00000001:0.0:1787829923.709364:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829923.709369:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298744492
00000100:00000010:0.0:1787829923.709371:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc060.
00000100:00000001:0.0:1787829923.709374:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829923.709377:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829923.709379:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709381:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829923.709385:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923]  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829923.709391:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829923.709393:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829923.709394:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829923.709396:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829923.709397:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829923.709398:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829923.709400:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829923.709405:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709409:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829923.709410:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709411:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709413:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829923.709415:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709418:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829923.709421:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709425:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829923.709427:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709428:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709429:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709430:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829923.709431:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829923.709433:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829923.709435:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829923.709437:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829923.709439:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829923.709443:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327392640:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829923.709447:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709448:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.709450:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327392640/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709453:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829923.709455:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829923.709457:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829923.709459:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829923.709460:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829923.709460:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829923.709462:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829923.709463:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4c00.
02000000:00000001:0.0:1787829923.709466:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829923.709467:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829923.709468:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829923.709470:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829923.709471:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709472:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709476:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829923.709477:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709479:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829923.709480:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829923.709481:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829923.709483:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709487:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829923.709488:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709489:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829923.709490:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829923.709491:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829923.709493:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392768:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829923.709494:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829923.709495:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829923.709496:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709498:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829923.709499:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829923.709500:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709502:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829923.709504:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829923.709506:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829923.709507:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392768, portal 10
00000100:00000001:0.0:1787829923.709508:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829923.709509:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829923.709513:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709517:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829923.709518:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829923.709520:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829923.709521:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327392768, offset 0
00000400:00000200:0.0:1787829923.709523:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709525:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829923.709527:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829923.709529:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829923.709530:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829923.709531:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829923.709533:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709535:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40840:40840:256:4294967295] 192.168.203.116@tcp LPNI seq info [40840:40840:8:4294967295]
00000400:00000200:0.0:1787829923.709537:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829923.709541:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829923.709543:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829923.709545:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829923.709547:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829923.709548:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829923.709551:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829923.709553:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829923.709555:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829923.709556:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829923.709558:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829923.709560:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829923.709562:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829923.709563:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829923.709564:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829923.709566:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709570:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.709572:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709575:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709576:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829923.709578:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829923.709580:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829923.709582:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829923.709583:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298743818
00000100:00000001:0.0:1787829923.709585:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298744492
00000100:00000001:0.0:1787829923.709586:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298743818
00000100:00000001:0.0:1787829923.709587:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298744492
00000100:00000001:0.0:1787829923.709588:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829923.709589:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709590:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829923.709592:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923]  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829923.709596:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829923.709597:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829923.709597:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829923.709599:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829923.709600:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829923.709601:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829923.709603:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829923.709608:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709611:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829923.709612:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709613:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709613:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829923.709615:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709619:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829923.709620:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709624:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829923.709625:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709626:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709627:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709627:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829923.709629:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829923.709630:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829923.709632:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829923.709633:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829923.709635:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829923.709639:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392768:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829923.709640:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829923.709642:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709650:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829923.709651:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709652:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829923.709653:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829923.709654:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829923.709671:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392896:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829923.709673:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829923.709674:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829923.709675:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709676:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829923.709678:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829923.709679:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709681:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829923.709682:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829923.709684:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829923.709686:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327392896, portal 4
00000100:00000001:0.0:1787829923.709687:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829923.709688:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829923.709691:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709694:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829923.709696:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829923.709697:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829923.709698:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327392896, offset 0
00000400:00000200:0.0:1787829923.709700:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709702:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829923.709704:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829923.709705:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829923.709707:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829923.709708:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829923.709709:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709712:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40841:40841:256:4294967295] 192.168.203.116@tcp LPNI seq info [40841:40841:8:4294967295]
00000400:00000200:0.0:1787829923.709714:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829923.709718:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829923.709720:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829923.709722:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829923.709724:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829923.709726:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829923.709728:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829923.709729:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829923.709732:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829923.709733:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829923.709735:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829923.709737:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829923.709738:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829923.709739:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829923.709741:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829923.709742:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709747:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.709749:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709753:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709754:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829923.709755:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829923.709757:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829923.709764:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829923.709765:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298743826
00000100:00000001:0.0:1787829923.709767:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298744492
00000100:00000001:0.0:1787829923.709768:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298743826
00000100:00000001:0.0:1787829923.709769:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298744492
00000100:00000001:0.0:1787829923.709771:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829923.709772:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709773:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829923.709776:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923]  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829923.709780:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829923.709781:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829923.709782:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829923.709784:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829923.709784:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829923.709785:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829923.709787:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829923.709791:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709795:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829923.709796:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709797:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709798:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829923.709800:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709803:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829923.709805:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709808:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829923.709810:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709810:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709811:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709812:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829923.709813:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829923.709814:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829923.709816:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829923.709817:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829923.709820:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829923.709824:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327392896:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829923.709826:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829923.709828:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709831:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829923.709832:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709833:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829923.709834:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829923.709835:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829923.709836:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393024:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829923.709838:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829923.709838:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829923.709839:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829923.709840:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829923.709842:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829923.709843:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829923.709844:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829923.709846:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829923.709847:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829923.709849:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393024, portal 4
00000100:00000001:0.0:1787829923.709850:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829923.709851:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:0.0:1787829923.709854:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709857:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829923.709859:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829923.709860:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829923.709862:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327393024, offset 0
00000400:00000200:0.0:1787829923.709864:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709865:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829923.709867:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829923.709869:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829923.709870:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829923.709872:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829923.709873:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829923.709875:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40842:40842:256:4294967295] 192.168.203.116@tcp LPNI seq info [40842:40842:8:4294967295]
00000400:00000200:0.0:1787829923.709877:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829923.709881:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829923.709883:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829923.709884:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829923.709886:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829923.709888:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787829923.709890:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829923.709892:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787829923.709894:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829923.709895:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829923.709897:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829923.709898:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829923.709900:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829923.709901:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829923.709902:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829923.709904:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709908:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.709910:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709914:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709915:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829923.709916:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829923.709918:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829923.709920:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829923.709921:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298743831
00000100:00000001:0.0:1787829923.709922:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298744492
00000100:00000001:0.0:1787829923.709923:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298743831
00000100:00000001:0.0:1787829923.709925:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298744492
00000100:00000001:0.0:1787829923.709926:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829923.709927:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.709928:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829923.709929:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923]  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829923.709933:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829923.709934:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829923.709934:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829923.709936:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829923.709936:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829923.709937:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829923.709939:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829923.709944:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709947:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829923.709948:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709948:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709949:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829923.709951:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709954:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829923.709956:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709960:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829923.709962:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709962:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829923.709963:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709964:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829923.709965:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829923.709966:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829923.709968:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829923.709969:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829923.709982:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829923.709986:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393024:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829923.709988:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.709989:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.709990:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327392768/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.709994:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829923.709995:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829923.709996:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829923.709997:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829923.709998:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829923.709999:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829923.710000:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829923.710001:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4800.
02000000:00000001:0.0:1787829923.710003:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829923.710003:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829923.710004:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829923.710006:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829923.710007:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.710008:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.710010:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327392896/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.710013:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829923.710014:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829923.710015:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829923.710016:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829923.710016:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829923.710017:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829923.710018:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829923.710019:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4400.
02000000:00000001:0.0:1787829923.710020:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829923.710020:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829923.710021:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829923.710022:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829923.710023:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.710024:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829923.710025:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327393024/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829923.710028:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829923.710029:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829923.710030:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829923.710031:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829923.710032:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829923.710032:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829923.710033:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829923.710034:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7c00.
02000000:00000001:0.0:1787829923.710035:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829923.710035:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829923.710036:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829923.710037:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829923.710038:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.710039:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829923.710040:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829923.710041:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829923.710042:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829923.710043:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829924.220749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829924.220754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829924.220756:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829924.220759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829924.220762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829924.220766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829924.220769:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829924.220771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829924.220774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829924.412743:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298744492, at 4298744512
00000100:00000040:3.0:1787829924.412743:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298744492, at 4298744512
00000100:00000010:3.0:1787829924.412764:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc060.
00000100:00000010:2.0:1787829924.412768:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc540.
00010000:00000001:0.0:1787829924.540772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829924.540778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829924.540785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829924.540787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829924.540790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829924.540791:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829924.540843:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829924.540846:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829924.540849:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829925.244773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829925.244779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829925.244782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829925.244787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829925.244791:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829925.244794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829925.244797:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829925.244799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829925.244803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564730:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829925.564735:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829925.564737:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829925.564739:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564742:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829925.564744:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564745:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564746:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564748:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829925.564749:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829925.564750:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829925.564751:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564752:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829925.564754:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564755:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564756:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829925.564758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829925.564759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829925.564812:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829925.564816:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829925.564819:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829926.268750:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829926.268758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829926.268760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829926.268765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829926.268768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829926.268771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829926.268773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829926.268776:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829926.268778:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829926.588799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829926.588804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829926.588807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829926.588808:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829926.588810:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829926.588811:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829926.588823:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829926.588825:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829926.588827:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829927.292755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829927.292760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829927.292761:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829927.292765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829927.292767:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829927.292770:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829927.292772:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829927.292773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829927.292775:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829927.612748:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829927.612754:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829927.612758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829927.612759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829927.612761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829927.612762:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829927.612826:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829927.612830:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829927.612834:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829928.316713:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829928.316718:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829928.316719:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829928.316723:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829928.316725:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829928.316727:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829928.316729:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829928.316731:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829928.316732:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829928.636753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829928.636758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829928.636762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829928.636763:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829928.636765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829928.636766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829928.636815:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829928.636818:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829928.636820:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829928.828798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829928.828808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829928.828810:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829928.828814:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829928.828818:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829928.828824:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4076
00000400:00000200:0.0:1787829928.828832:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829928.828838:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829928.828841:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829928.828844:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.828847:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829928.828849:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.828851:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829928.828854:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829928.828856:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829928.828859:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829928.828862:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829928.828864:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829928.828871:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829928.828874:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.828876:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829928.828878:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.828879:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829928.828882:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829928.828885:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829928.828887:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829928.828889:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829928.828892:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829928.828894:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.828896:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829928.828898:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829928.828906:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716da40.
00000020:00000040:0.0:1787829928.828909:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829928.828913:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829928.828915:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.828917:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829928.828920:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829928.828923:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829928.828949:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532800.
00000100:00000001:0.0:1787829928.828956:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829928.828964:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829928.828973:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829928.828977:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829928.828981:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000226d20b5] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829928.829001:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829928.829004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829928.829007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829928.829008:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829928.829010:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829928.829012:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829928.829014:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4076
00000400:00000200:0.0:1787829928.829018:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829928.829021:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829928.829024:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829928.829027:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829029:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829928.829032:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829928.829032:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829928.829037:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829928.829038:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787829928.829040:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829928.829043:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000100:00000001:1.0:1787829928.829043:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787829928.829045:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829928.829048:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829928.829052:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829928.829053:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787829928.829055:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:0.0:1787829928.829061:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829065:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829928.829069:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829928.829070:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829928.829074:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829928.829077:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:1.0:1787829928.829078:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829928.829081:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829928.829083:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787829928.829084:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.829087:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829928.829088:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829928.829092:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829928.829094:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787829928.829098:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327393152:192.168.203.116@tcp:250:kworker.0
02000000:00000001:0.0:1787829928.829099:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829103:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829928.829105:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1787829928.829108:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829928.829111:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:0.0:1787829928.829112:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829115:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.829115:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.829120:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:0.0:1787829928.829122:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716e580.
00000020:00000040:0.0:1787829928.829126:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000010:1.0:1787829928.829127:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829928.829129:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829928.829131:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829928.829132:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829136:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829928.829137:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787829928.829140:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:0.0:1787829928.829141:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829928.829144:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
02000000:00000010:0.0:1787829928.829148:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530c00.
00000400:00000010:1.0:1787829928.829151:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000001:0.0:1787829928.829152:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787829928.829157:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393152, portal 25
00000100:00100000:0.0:1787829928.829160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829928.829161:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829928.829163:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419573824 : -116696289977792 : ffff95dd8716da40)
00000100:00000040:1.0:1787829928.829171:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829928.829175:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829928.829178:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829928.829184:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc9f2c41] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829189:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829928.829195:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829199:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829928.829199:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829928.829202:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327393152, offset 0
00000100:00080000:0.0:1787829928.829203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829928.829208:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:0.0:1787829928.829209:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1787829928.829212:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829928.829214:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829928.829216:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829928.829218:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829928.829223:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4076
00000400:00000200:1.0:1787829928.829224:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829928.829234:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829928.829241:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829928.829246:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829928.829248:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829928.829252:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40843:40843:256:4294967295] 192.168.203.116@tcp LPNI seq info [40843:40843:8:4294967295]
00000400:00000200:1.0:1787829928.829257:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829928.829262:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829928.829269:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:1.0:1787829928.829272:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787829928.829273:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829928.829280:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829283:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787829928.829285:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000040:0.0:1787829928.829287:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829291:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000010:1.0:1787829928.829292:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000020:00000001:0.0:1787829928.829293:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829928.829296:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:0.0:1787829928.829298:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.0:1787829928.829301:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:0.0:1787829928.829302:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:1.0:1787829928.829305:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1787829928.829306:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:0.0:1787829928.829310:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:1.1:1787829928.829311:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829928.829314:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1787829928.829316:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829320:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:1.0:1787829928.829321:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000040:0.0:1787829928.829324:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000010:1.0:1787829928.829324:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000001:0.0:1787829928.829329:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829928.829329:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1787829928.829335:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:1.0:1787829928.829335:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829928.829342:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000100:00080000:0.0:1787829928.829343:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000010:1.0:1787829928.829347:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:0.0:1787829928.829348:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829352:0:2346:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1787829928.829353:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829928.829358:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1787829928.829361:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829928.829362:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829366:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829928.829369:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829928.829374:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829376:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829928.829380:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f0c0.
00000020:00000040:0.0:1787829928.829382:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000100:00000001:1.0:1787829928.829385:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787829928.829386:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829928.829389:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829928.829390:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829928.829393:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829928.829396:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829928.829399:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829405:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829928.829406:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531800.
00000100:00000001:1.0:1787829928.829408:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829928.829411:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829416:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829928.829419:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787829928.829424:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829928.829429:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298749612
00000100:00000001:0.0:1787829928.829434:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829928.829436:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829928.829443:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63de0.
00000100:00000040:0.0:1787829928.829444:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003ce953a3] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829452:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829928.829455:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829928.829459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787829928.829460:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298749612
00000100:00080000:0.0:1787829928.829465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:1.0:1787829928.829467:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63420.
00000100:00000001:0.0:1787829928.829469:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829928.829472:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829928.829473:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829928.829477:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787829928.829479:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787829928.829482:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4076
00000100:00000001:1.0:1787829928.829484:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829928.829489:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829928.829489:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787829928.829497:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00100000:1.0:1787829928.829498:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928]  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787829928.829500:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829928.829508:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000001:1.0:1787829928.829511:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787829928.829512:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:1.0:1787829928.829515:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1787829928.829517:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:1.0:1787829928.829518:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000100:00000001:0.0:1787829928.829521:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829928.829523:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000020:00000001:0.0:1787829928.829526:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:1.0:1787829928.829526:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000020:00000040:0.0:1787829928.829529:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:1.0:1787829928.829530:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:0.0:1787829928.829533:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787829928.829535:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787829928.829537:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:0.0:1787829928.829541:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829928.829545:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:0.0:1787829928.829550:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000200:1.0:1787829928.829551:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829928.829552:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829928.829555:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829558:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829928.829562:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829928.829566:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829928.829569:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829569:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1787829928.829573:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829928.829573:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829928.829575:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829928.829578:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829578:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829928.829581:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829928.829582:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1787829928.829583:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829928.829586:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829587:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829928.829590:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1787829928.829592:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716c000.
00000020:00000040:0.0:1787829928.829596:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:0.0:1787829928.829600:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829928.829602:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829928.829605:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787829928.829605:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787829928.829608:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829928.829610:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829928.829612:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787829928.829614:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532c00.
00000100:00000001:0.0:1787829928.829619:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829620:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829928.829624:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829928.829625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829928.829626:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.829628:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829633:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829928.829635:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829928.829636:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829928.829640:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829928.829640:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829928.829647:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829928.829649:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829928.829653:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cdd0ad0f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829928.829684:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829928.829686:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829928.829689:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4086)
00000100:00100000:1.0:1787829928.829700:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327393152:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829928.829706:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829709:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.829711:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716da40 x1874671327393152/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829716:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829928.829718:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829928.829720:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829928.829722:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829928.829723:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829928.829724:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829928.829726:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829928.829727:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532800.
02000000:00000001:1.0:1787829928.829729:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829928.829731:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829928.829732:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716da40.
00000100:00000001:1.0:1787829928.829734:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829928.829734:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829736:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829741:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829928.829742:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829743:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829928.829744:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829928.829745:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829928.829747:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829755:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829928.829756:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.829757:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829928.829759:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829928.829760:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829928.829761:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393280:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829928.829763:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829928.829764:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829928.829765:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.829766:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829928.829768:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829928.829769:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829928.829773:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829928.829774:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829928.829777:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829928.829779:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393280, portal 10
00000100:00000001:1.0:1787829928.829780:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829928.829781:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419576704 : -116696289974912 : ffff95dd8716e580)
00000100:00000040:1.0:1787829928.829785:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829790:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829928.829792:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829928.829794:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829928.829796:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327393280, offset 0
00000400:00000200:1.0:1787829928.829800:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829928.829804:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829928.829808:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829928.829811:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829928.829813:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829928.829815:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829928.829818:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829928.829821:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40844:40844:256:4294967295] 192.168.203.116@tcp LPNI seq info [40844:40844:8:4294967295]
00000400:00000200:1.0:1787829928.829825:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829928.829831:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829928.829835:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829928.829839:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829928.829843:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829928.829844:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829928.829847:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829928.829849:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829928.829851:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829928.829853:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829928.829854:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829928.829857:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829928.829859:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829928.829860:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829928.829861:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829928.829863:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829868:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.829870:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829874:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829875:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829928.829877:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829928.829880:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829928.829882:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829928.829883:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298748939
00000100:00000001:1.0:1787829928.829886:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298749612
00000100:00000001:1.0:1787829928.829888:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298748939
00000100:00000001:1.0:1787829928.829890:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298749612
00000100:00000001:1.0:1787829928.829892:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829928.829894:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.829897:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829928.829900:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928]  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829928.829907:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829928.829909:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829928.829911:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829928.829913:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829928.829914:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829928.829916:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829928.829919:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829928.829927:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829933:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829928.829951:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829953:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.829955:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829928.829958:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829965:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829928.829969:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.829975:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829928.829978:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829979:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.829980:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.829982:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829928.829983:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829928.829986:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829928.829989:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829928.829991:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829928.829994:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829928.830001:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393280:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829928.830005:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829928.830008:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830014:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829928.830015:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.830017:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829928.830019:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829928.830021:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829928.830024:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393408:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829928.830027:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829928.830028:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829928.830029:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.830031:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829928.830034:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829928.830036:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829928.830039:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829928.830041:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829928.830044:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829928.830046:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393408, portal 4
00000100:00000001:1.0:1787829928.830048:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829928.830049:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419579584 : -116696289972032 : ffff95dd8716f0c0)
00000100:00000040:1.0:1787829928.830055:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830061:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829928.830064:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829928.830066:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829928.830068:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327393408, offset 0
00000400:00000200:1.0:1787829928.830072:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829928.830076:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829928.830080:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829928.830082:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829928.830084:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829928.830086:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829928.830088:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829928.830091:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40845:40845:256:4294967295] 192.168.203.116@tcp LPNI seq info [40845:40845:8:4294967295]
00000400:00000200:1.0:1787829928.830095:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829928.830100:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829928.830104:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829928.830107:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829928.830111:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829928.830113:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829928.830117:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829928.830120:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829928.830123:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829928.830125:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829928.830128:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829928.830131:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829928.830133:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829928.830135:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829928.830137:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829928.830140:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830147:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.830150:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830155:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.830157:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829928.830159:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829928.830162:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829928.830165:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829928.830167:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298748946
00000100:00000001:1.0:1787829928.830169:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298749612
00000100:00000001:1.0:1787829928.830171:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298748946
00000100:00000001:1.0:1787829928.830172:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298749612
00000100:00000001:1.0:1787829928.830174:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829928.830176:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.830178:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829928.830181:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928]  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829928.830187:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829928.830188:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829928.830190:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829928.830192:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829928.830193:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829928.830195:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829928.830198:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829928.830205:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830211:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829928.830212:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830214:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.830215:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829928.830218:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830223:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829928.830226:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830233:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829928.830235:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830236:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.830237:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830239:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829928.830240:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829928.830243:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829928.830245:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829928.830247:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829928.830252:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829928.830259:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393408:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829928.830263:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829928.830266:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830271:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829928.830272:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.830274:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829928.830276:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829928.830277:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829928.830279:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393536:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829928.830283:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829928.830284:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829928.830286:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829928.830287:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829928.830290:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829928.830292:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829928.830295:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829928.830297:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829928.830300:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829928.830303:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393536, portal 4
00000100:00000001:1.0:1787829928.830304:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829928.830306:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419567104 : -116696289984512 : ffff95dd8716c000)
00000100:00000040:1.0:1787829928.830310:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830317:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829928.830319:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829928.830321:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829928.830323:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327393536, offset 0
00000400:00000200:1.0:1787829928.830327:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829928.830331:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829928.830334:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829928.830337:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829928.830339:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829928.830341:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829928.830343:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829928.830345:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40846:40846:256:4294967295] 192.168.203.116@tcp LPNI seq info [40846:40846:8:4294967295]
00000400:00000200:1.0:1787829928.830349:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829928.830354:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829928.830358:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829928.830361:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829928.830365:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829928.830367:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787829928.830371:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829928.830374:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787829928.830377:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829928.830379:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829928.830381:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829928.830384:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829928.830387:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829928.830389:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829928.830391:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829928.830395:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830402:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.830404:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830410:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.830411:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829928.830415:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829928.830417:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829928.830419:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829928.830421:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298748951
00000100:00000001:1.0:1787829928.830423:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298749612
00000100:00000001:1.0:1787829928.830425:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298748951
00000100:00000001:1.0:1787829928.830427:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298749612
00000100:00000001:1.0:1787829928.830429:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829928.830431:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.830433:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829928.830436:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928]  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829928.830443:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829928.830444:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829928.830446:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829928.830448:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829928.830449:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829928.830451:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829928.830453:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829928.830461:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830472:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829928.830473:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830475:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.830476:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829928.830480:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830485:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829928.830489:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830495:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829928.830497:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830499:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829928.830500:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830501:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829928.830503:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829928.830505:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829928.830508:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829928.830510:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829928.830513:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829928.830521:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393536:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829928.830524:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830527:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.830529:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716e580 x1874671327393280/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830535:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829928.830536:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829928.830539:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829928.830541:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829928.830542:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829928.830544:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829928.830545:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829928.830547:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530c00.
02000000:00000001:1.0:1787829928.830549:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829928.830550:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829928.830552:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716e580.
00000100:00000001:1.0:1787829928.830554:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829928.830555:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830557:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.830559:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f0c0 x1874671327393408/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830565:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829928.830566:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829928.830567:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829928.830569:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829928.830570:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829928.830571:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829928.830573:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829928.830574:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531800.
02000000:00000001:1.0:1787829928.830577:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829928.830577:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829928.830578:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f0c0.
00000100:00000001:1.0:1787829928.830580:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829928.830582:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830583:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829928.830586:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716c000 x1874671327393536/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829928.830592:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829928.830593:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829928.830594:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:1.0:1787829928.830596:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829928.830597:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829928.830598:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829928.830601:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829928.830602:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532c00.
02000000:00000001:1.0:1787829928.830604:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829928.830605:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829928.830606:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716c000.
00000100:00000001:1.0:1787829928.830608:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829928.830609:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830611:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829928.830615:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829928.830616:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829928.830618:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829928.830619:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829929.340769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829929.340775:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829929.340777:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829929.340782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829929.340785:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829929.340788:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829929.340791:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829929.340793:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829929.340796:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829929.532744:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298749612, at 4298749632
00000100:00000040:2.0:1787829929.532746:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298749612, at 4298749632
00000100:00000010:3.0:1787829929.532757:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63420.
00000100:00000010:2.0:1787829929.532759:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63de0.
00010000:00000001:0.0:1787829929.660761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829929.660768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829929.660774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829929.660776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829929.660779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829929.660780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829929.660800:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829929.660802:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829929.660805:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829930.364792:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829930.364799:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829930.364801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829930.364806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829930.364810:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829930.364813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829930.364816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829930.364819:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829930.364822:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829930.684765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829930.684772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829930.684777:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829930.684778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829930.684781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829930.684784:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829930.684787:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829930.684789:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829930.684793:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829930.684795:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829930.684796:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829930.684799:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829930.684846:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829930.684849:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829930.684851:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829931.388827:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829931.388832:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829931.388833:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829931.388836:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829931.388839:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829931.388841:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829931.388843:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829931.388844:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829931.388846:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829931.708762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829931.708768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829931.708774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829931.708775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829931.708778:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829931.708779:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829931.708824:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829931.708827:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829931.708829:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829932.412755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829932.412762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829932.412764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829932.412770:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829932.412774:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829932.412777:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829932.412779:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829932.412782:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829932.412785:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829932.732761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829932.732767:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829932.732772:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829932.732774:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829932.732777:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829932.732778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829932.732796:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829932.732798:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829932.732801:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829933.436818:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829933.436826:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829933.436829:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829933.436836:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829933.436841:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829933.436845:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.436848:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829933.436852:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.436856:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829933.756765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829933.756770:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829933.756774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829933.756775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829933.756777:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829933.756778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829933.756862:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829933.756869:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829933.756874:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829933.948813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829933.948822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829933.948825:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829933.948829:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829933.948834:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829933.948841:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4081
00000400:00000200:1.0:1787829933.948850:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829933.948856:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829933.948859:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829933.948861:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.948864:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.948865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.948867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829933.948870:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829933.948872:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829933.948874:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829933.948877:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829933.948879:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829933.948881:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829933.948882:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.948884:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.948886:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.948887:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829933.948890:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829933.948892:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829933.948894:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829933.948897:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829933.948914:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829933.948916:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.948918:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829933.948920:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829933.948930:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000020:00000040:1.0:1787829933.948933:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829933.948937:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829933.948938:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.948941:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829933.948944:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829933.948948:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829933.948955:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6c00.
00000100:00000001:1.0:1787829933.948963:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829933.948975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829933.948986:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829933.948991:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829933.948997:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829933.949028:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829933.949030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829933.949033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829933.949034:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829933.949035:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829933.949037:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829933.949040:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4081
00000400:00000200:1.0:1787829933.949043:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829933.949045:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829933.949046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829933.949048:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949050:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.949052:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949055:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829933.949058:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829933.949060:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829933.949062:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829933.949064:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829933.949067:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829933.949069:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829933.949072:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829933.949075:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829933.949075:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.949079:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829933.949085:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829933.949085:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829933.949090:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787829933.949092:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829933.949098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949102:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829933.949104:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787829933.949106:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829933.949111:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829933.949117:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949124:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829933.949131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829933.949133:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:1.0:1787829933.949135:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.949138:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949138:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829933.949141:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000020:00000040:1.0:1787829933.949145:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829933.949146:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829933.949149:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829933.949151:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949152:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829933.949153:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829933.949155:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829933.949157:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5400.
00000100:00000001:1.0:1787829933.949161:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.949164:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787829933.949170:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829933.949171:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829933.949184:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327393664:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829933.949196:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829933.949197:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787829933.949202:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.949204:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829933.949209:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829933.949211:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829933.949214:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829933.949225:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000100:00000001:1.0:1787829933.949225:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829933.949230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1787829933.949232:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829933.949242:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829933.949244:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829933.949248:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829933.949249:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829933.949257:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1787829933.949259:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00080000:1.0:1787829933.949265:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4081
00000400:00000010:0.0:1787829933.949270:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000400:00000200:1.0:1787829933.949276:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829933.949282:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393664, portal 25
00000400:00000200:1.0:1787829933.949283:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829933.949290:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829933.949292:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829933.949295:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:1.0:1787829933.949297:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949302:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.949308:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829933.949309:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.949319:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787829933.949322:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829933.949325:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829933.949330:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000020:00000001:1.0:1787829933.949331:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829933.949335:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000100:00000200:0.0:1787829933.949336:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327393664, offset 0
00000020:00000001:1.0:1787829933.949338:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829933.949344:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829933.949350:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829933.949351:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000001:1.0:1787829933.949357:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829933.949364:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:1.0:1787829933.949365:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949370:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.949374:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829933.949376:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:1.0:1787829933.949378:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829933.949383:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829933.949389:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829933.949390:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787829933.949392:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829933.949395:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829933.949398:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829933.949399:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
02000000:00000001:1.0:1787829933.949404:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949408:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829933.949409:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000008:00100000:1.0:1787829933.949413:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829933.949414:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000008:00000001:1.0:1787829933.949419:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829933.949427:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40847:40847:256:4294967295] 192.168.203.116@tcp LPNI seq info [40847:40847:8:4294967295]
00000100:00000001:1.0:1787829933.949427:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829933.949437:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
00000400:00000200:0.0:1787829933.949443:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:1.0:1787829933.949444:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829933.949451:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829933.949454:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949461:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829933.949463:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1787829933.949467:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1787829933.949468:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829933.949477:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6400.
00000800:00000200:0.0:1787829933.949480:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949484:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829933.949487:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00100000:1.0:1787829933.949494:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829933.949501:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829933.949512:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1787829933.949514:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829933.949519:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829933.949527:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:1.0:1787829933.949528:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1787829933.949535:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829933.949552:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949552:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787829933.949557:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000100:00080000:1.0:1787829933.949559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829933.949565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829933.949566:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1787829933.949567:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1787829933.949569:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829933.949575:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787829933.949579:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1787829933.949582:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4081
00000400:00000200:0.0:1787829933.949587:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829933.949592:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829933.949597:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:1.0:1787829933.949600:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829933.949605:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829933.949609:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829933.949616:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829933.949626:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949634:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829933.949638:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.949645:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829933.949645:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829933.949682:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.949684:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829933.949690:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829933.949694:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829933.949697:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298754733
00000100:00000010:0.0:1787829933.949703:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc1e0.
00000100:00000001:0.0:1787829933.949711:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829933.949717:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298754733
00000100:00000010:0.0:1787829933.949721:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc720.
00000100:00000001:0.0:1787829933.949725:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829933.949734:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829933.949740:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829933.949742:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829933.949744:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1787829933.949745:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787829933.949751:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829933.949753:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787829933.949757:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829933.949759:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:1.0:1787829933.949763:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00100000:0.0:1787829933.949770:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933]  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829933.949773:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949781:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829933.949788:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829933.949793:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000001:0.0:1787829933.949794:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829933.949799:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:1.0:1787829933.949801:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000400:00000010:0.0:1787829933.949803:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000100:00080000:1.0:1787829933.949808:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829933.949810:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000100:00000001:1.0:1787829933.949813:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829933.949815:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
02000000:00000001:1.0:1787829933.949818:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829933.949820:0:2346:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:1.0:1787829933.949824:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1787829933.949829:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787829933.949831:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949837:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829933.949842:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829933.949851:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829933.949855:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829933.949858:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787829933.949861:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829933.949869:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829933.949875:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829933.949880:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:1.0:1787829933.949880:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.949886:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829933.949886:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829933.949889:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829933.949892:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:1.0:1787829933.949893:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.949896:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:1.0:1787829933.949913:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5000.
00000100:00100000:0.0:1787829933.949918:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829933.949921:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.949935:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829933.949936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829933.949950:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829933.949953:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829933.949957:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829933.949967:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.949974:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829933.949983:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.949989:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829933.949993:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.949994:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950003:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:1.0:1787829933.950006:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4091)
00000100:00000001:0.0:1787829933.950007:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829933.950017:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829933.950023:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829933.950025:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829933.950031:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829933.950041:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327393664:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829933.950048:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950050:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950054:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327393664/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950060:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829933.950062:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950065:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829933.950068:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829933.950071:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829933.950072:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829933.950075:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829933.950077:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6c00.
02000000:00000001:0.0:1787829933.950081:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829933.950083:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829933.950084:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829933.950088:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829933.950089:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950091:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950097:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829933.950099:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950102:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829933.950104:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829933.950105:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829933.950110:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950117:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829933.950119:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.950121:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829933.950123:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829933.950126:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829933.950128:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393792:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829933.950132:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829933.950134:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829933.950135:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.950137:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950140:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829933.950142:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829933.950146:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829933.950149:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829933.950152:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829933.950155:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393792, portal 10
00000100:00000001:0.0:1787829933.950157:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829933.950158:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829933.950165:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950171:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829933.950174:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829933.950176:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829933.950178:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327393792, offset 0
00000400:00000200:0.0:1787829933.950182:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829933.950186:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829933.950190:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829933.950193:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829933.950196:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829933.950198:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829933.950200:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829933.950204:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40848:40848:256:4294967295] 192.168.203.116@tcp LPNI seq info [40848:40848:8:4294967295]
00000400:00000200:0.0:1787829933.950207:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829933.950214:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829933.950218:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829933.950222:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829933.950225:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829933.950228:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829933.950232:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829933.950235:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829933.950238:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829933.950240:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829933.950243:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829933.950246:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829933.950249:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829933.950251:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829933.950253:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829933.950257:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950264:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950267:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950273:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950274:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829933.950277:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829933.950280:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829933.950283:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829933.950285:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298754059
00000100:00000001:0.0:1787829933.950287:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298754733
00000100:00000001:0.0:1787829933.950289:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298754059
00000100:00000001:0.0:1787829933.950291:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298754733
00000100:00000001:0.0:1787829933.950293:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829933.950295:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950297:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829933.950300:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933]  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829933.950307:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829933.950308:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829933.950310:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829933.950313:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829933.950314:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829933.950315:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829933.950318:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829933.950326:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950332:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829933.950333:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950334:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829933.950336:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829933.950339:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950345:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829933.950349:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950355:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829933.950357:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950359:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829933.950360:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950362:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829933.950364:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829933.950366:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829933.950370:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829933.950372:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829933.950376:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829933.950384:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393792:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829933.950388:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829933.950391:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950397:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829933.950399:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.950401:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829933.950403:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829933.950405:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829933.950407:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393920:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829933.950410:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829933.950412:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829933.950414:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.950415:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950419:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829933.950420:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829933.950423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829933.950426:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829933.950429:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829933.950431:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327393920, portal 4
00000100:00000001:0.0:1787829933.950433:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829933.950434:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829933.950440:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950446:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829933.950449:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829933.950451:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829933.950453:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327393920, offset 0
00000400:00000200:0.0:1787829933.950457:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829933.950460:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829933.950464:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829933.950466:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829933.950469:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829933.950471:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829933.950473:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829933.950476:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40849:40849:256:4294967295] 192.168.203.116@tcp LPNI seq info [40849:40849:8:4294967295]
00000400:00000200:0.0:1787829933.950479:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829933.950483:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829933.950486:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829933.950488:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829933.950491:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829933.950492:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829933.950495:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829933.950496:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829933.950499:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829933.950500:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829933.950502:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829933.950504:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829933.950506:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829933.950507:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829933.950508:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829933.950511:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950518:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950521:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950527:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950528:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829933.950532:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829933.950535:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829933.950538:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829933.950539:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298754066
00000100:00000001:0.0:1787829933.950542:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298754733
00000100:00000001:0.0:1787829933.950544:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298754066
00000100:00000001:0.0:1787829933.950546:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298754733
00000100:00000001:0.0:1787829933.950548:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829933.950550:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950552:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829933.950556:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933]  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829933.950563:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829933.950565:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829933.950566:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829933.950567:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829933.950568:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829933.950570:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829933.950572:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829933.950577:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950581:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829933.950581:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950582:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829933.950583:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829933.950585:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950588:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829933.950591:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950595:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829933.950597:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950598:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829933.950598:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950599:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829933.950600:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829933.950601:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829933.950603:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829933.950604:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829933.950606:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829933.950610:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327393920:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829933.950612:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829933.950614:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950618:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829933.950619:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.950620:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829933.950621:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829933.950622:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829933.950624:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394048:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829933.950625:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829933.950627:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829933.950628:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829933.950629:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950631:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829933.950632:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829933.950634:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829933.950636:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829933.950637:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829933.950639:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394048, portal 4
00000100:00000001:0.0:1787829933.950640:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829933.950641:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00000040:0.0:1787829933.950644:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950648:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829933.950650:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829933.950651:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829933.950652:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327394048, offset 0
00000400:00000200:0.0:1787829933.950671:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829933.950674:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829933.950676:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829933.950678:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829933.950680:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829933.950681:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829933.950682:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829933.950684:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40850:40850:256:4294967295] 192.168.203.116@tcp LPNI seq info [40850:40850:8:4294967295]
00000400:00000200:0.0:1787829933.950687:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829933.950690:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829933.950693:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829933.950695:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829933.950698:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829933.950699:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787829933.950702:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829933.950703:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787829933.950705:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829933.950707:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829933.950708:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829933.950710:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829933.950712:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829933.950713:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829933.950715:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829933.950717:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950722:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950724:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950727:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950728:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829933.950730:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829933.950732:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829933.950734:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829933.950735:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298754072
00000100:00000001:0.0:1787829933.950736:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298754733
00000100:00000001:0.0:1787829933.950738:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298754072
00000100:00000001:0.0:1787829933.950739:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298754733
00000100:00000001:0.0:1787829933.950740:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829933.950741:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950742:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829933.950744:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933]  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829933.950748:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829933.950748:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829933.950749:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829933.950750:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829933.950751:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829933.950752:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829933.950754:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829933.950758:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950765:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829933.950766:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950767:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829933.950768:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829933.950770:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950773:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829933.950775:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950778:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829933.950780:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950780:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829933.950781:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950782:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829933.950783:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829933.950784:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829933.950786:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829933.950787:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829933.950789:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829933.950793:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394048:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829933.950795:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950796:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950798:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327393792/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950801:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829933.950802:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950804:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829933.950805:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829933.950806:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829933.950807:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829933.950808:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829933.950809:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5400.
02000000:00000001:0.0:1787829933.950811:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829933.950812:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829933.950812:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829933.950814:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829933.950815:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950816:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950817:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327393920/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950821:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829933.950822:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950822:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530c00.
02000000:00000001:0.0:1787829933.950824:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829933.950824:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829933.950825:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829933.950826:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829933.950826:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6400.
02000000:00000001:0.0:1787829933.950827:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829933.950828:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829933.950829:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829933.950830:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829933.950831:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950832:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829933.950833:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327394048/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829933.950836:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829933.950837:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829933.950838:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532800.
02000000:00000001:0.0:1787829933.950839:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829933.950839:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829933.950840:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829933.950841:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829933.950842:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5000.
02000000:00000001:0.0:1787829933.950843:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829933.950844:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829933.950845:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829933.950846:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829933.950846:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950847:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829933.950849:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829933.950850:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829933.950851:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829933.950852:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829934.460769:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829934.460774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829934.460776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829934.460779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829934.460781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829934.460784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829934.460785:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829934.460787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829934.460788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829934.652809:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298754733, at 4298754752
00000100:00000010:2.0:1787829934.652823:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc1e0.
00000100:00000040:3.0:1787829934.652925:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298754733, at 4298754752
00000100:00000010:3.0:1787829934.652932:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc720.
00010000:00000001:0.0:1787829934.780774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829934.780782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829934.780788:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829934.780790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829934.780793:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829934.780795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829934.780854:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829934.780857:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829934.780860:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829935.484770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829935.484776:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829935.484778:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829935.484783:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829935.484787:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829935.484791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829935.484793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829935.484795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829935.484798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804863:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829935.804877:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829935.804879:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829935.804882:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804899:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829935.804901:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804903:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804905:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804909:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829935.804911:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829935.804912:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829935.804913:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804915:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829935.804916:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804918:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804919:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829935.804923:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829935.804924:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829935.804941:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829935.804943:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829935.804945:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829936.508787:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829936.508796:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829936.508798:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829936.508805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829936.508809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829936.508813:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829936.508816:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829936.508818:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829936.508821:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829936.828782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829936.828792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829936.828799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829936.828800:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829936.828804:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829936.828805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829936.828851:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829936.828854:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829936.828857:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829937.532756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829937.532763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829937.532764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829937.532769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829937.532772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829937.532774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829937.532776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829937.532778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829937.532779:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829937.852758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829937.852765:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829937.852770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829937.852771:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829937.852774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829937.852775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829937.852843:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829937.852852:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829937.852859:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829938.556754:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829938.556760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829938.556763:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829938.556767:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829938.556770:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829938.556774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829938.556776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829938.556778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829938.556781:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829938.876734:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829938.876739:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829938.876743:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829938.876744:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829938.876746:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829938.876747:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829938.876766:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829938.876768:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829938.876771:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829939.068739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829939.068747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829939.068750:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829939.068753:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829939.068756:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829939.068762:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4086
00000400:00000200:2.0:1787829939.068769:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829939.068774:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829939.068776:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829939.068778:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068780:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.068782:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068784:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829939.068786:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829939.068787:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829939.068789:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829939.068792:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829939.068793:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829939.068795:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829939.068796:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068798:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.068799:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068800:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829939.068802:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829939.068804:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068805:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829939.068807:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829939.068809:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829939.068811:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.068813:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829939.068815:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829939.068822:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115a40.
00000020:00000040:2.0:1787829939.068824:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829939.068827:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829939.068828:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.068831:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829939.068833:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829939.068837:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829939.068842:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829939.068847:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829939.068854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829939.068861:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829939.068874:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829939.068878:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370bb9a0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829939.068893:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829939.068895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829939.068897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829939.068898:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829939.068899:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829939.068904:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829939.068906:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4086
00000400:00000200:2.0:1787829939.068909:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829939.068910:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829939.068912:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829939.068913:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068915:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.068916:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068918:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829939.068920:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829939.068921:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829939.068923:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829939.068924:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829939.068926:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829939.068927:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829939.068929:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068930:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.068931:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.068933:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829939.068934:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787829939.068937:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829939.068940:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787829939.068940:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829939.068945:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787829939.068945:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829939.068949:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829939.068953:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787829939.068955:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829939.068956:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.068959:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829939.068964:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829939.068968:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.068970:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829939.068971:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.068975:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829939.068977:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117840.
02000000:00000001:0.0:1787829939.068979:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787829939.068980:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829939.068984:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829939.068984:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829939.068986:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.068990:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829939.068991:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829939.068996:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327394176:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787829939.068996:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829939.068999:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069003:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:2.0:1787829939.069004:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829939.069008:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069009:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829939.069013:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829939.069013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1787829939.069019:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829939.069026:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
00000100:00000001:2.0:1787829939.069026:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829939.069028:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069032:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829939.069033:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088c6579a] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829939.069040:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829939.069041:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829939.069045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787829939.069047:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00080000:2.0:1787829939.069048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829939.069051:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829939.069052:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829939.069054:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1787829939.069055:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00080000:2.0:1787829939.069057:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4086
00000400:00000200:2.0:1787829939.069061:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829939.069063:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394176, portal 25
00000400:00000200:2.0:1787829939.069064:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829939.069068:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829939.069069:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829939.069072:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402436160 : -116696307115456 : ffff95dd86115a40)
00000100:00000040:2.0:1787829939.069072:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069074:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.069075:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069077:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829939.069078:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829939.069079:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829939.069079:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829939.069081:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829939.069083:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829939.069085:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829939.069087:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069089:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.069091:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069094:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829939.069096:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829939.069097:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829939.069102:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069107:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829939.069111:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829939.069115:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829939.069119:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829939.069119:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829939.069123:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829939.069124:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000001:2.0:1787829939.069125:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829939.069129:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000200:0.0:1787829939.069130:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327394176, offset 0
00000008:00000001:2.0:1787829939.069131:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.069135:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829939.069138:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861161c0.
00000400:00000200:0.0:1787829939.069142:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000020:00000040:2.0:1787829939.069142:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829939.069146:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829939.069147:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.069149:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1787829939.069152:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829939.069152:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829939.069156:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829939.069160:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
02000000:00000010:2.0:1787829939.069162:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000400:00000200:0.0:1787829939.069166:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787829939.069167:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829939.069172:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00100000:2.0:1787829939.069176:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1787829939.069177:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829939.069182:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069189:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40851:40851:256:4294967295] 192.168.203.116@tcp LPNI seq info [40851:40851:8:4294967295]
00000100:00000001:2.0:1787829939.069193:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1787829939.069197:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829939.069198:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829939.069204:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000696263b6] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829939.069212:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829939.069216:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069224:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00000001:2.0:1787829939.069226:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829939.069230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1787829939.069234:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787829939.069234:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829939.069238:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.0:1787829939.069241:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000100:00080000:2.0:1787829939.069241:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829939.069247:0:36:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1787829939.069251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:2.0:1787829939.069254:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4086
00000800:00000200:0.1:1787829939.069256:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000400:00000200:2.0:1787829939.069262:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000800:00000100:0.0:1787829939.069267:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000400:00000200:2.0:1787829939.069270:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000800:00000010:0.0:1787829939.069271:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000100:00080000:2.0:1787829939.069273:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829939.069277:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1787829939.069279:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069284:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829939.069284:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829939.069288:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00000040:2.0:1787829939.069290:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829939.069291:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:2.0:1787829939.069296:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829939.069297:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:2.0:1787829939.069301:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:0.0:1787829939.069302:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829939.069303:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:0.0:1787829939.069311:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.069313:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787829939.069317:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829939.069320:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829939.069325:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829939.069325:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829939.069329:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829939.069330:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:2.0:1787829939.069336:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829939.069338:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.069339:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829939.069342:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069344:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:0.0:1787829939.069346:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787829939.069347:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829939.069352:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298759852
00000100:00080000:2.0:1787829939.069352:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829939.069355:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829939.069357:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:0.0:1787829939.069360:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc480.
02000000:00000001:2.0:1787829939.069360:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829939.069364:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.069366:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829939.069367:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1787829939.069369:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000008:00000001:2.0:1787829939.069370:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.069373:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069377:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298759852
00000100:00000010:2.0:1787829939.069377:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116d00.
00000020:00000040:2.0:1787829939.069380:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000010:0.0:1787829939.069384:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc300.
02000000:00000001:2.0:1787829939.069390:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787829939.069391:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:2.0:1787829939.069392:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829939.069395:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829939.069396:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829939.069397:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829939.069398:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069404:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829939.069404:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:0.0:1787829939.069409:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829939.069410:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829939.069415:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1787829939.069418:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829939/real 1787829939]  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829939.069424:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829939.069427:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829939.069434:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cdecb11] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829939.069458:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829939.069460:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1787829939.069461:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4096)
00000400:00000001:0.0:1787829939.069462:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829939.069465:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829939.069468:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829939.069469:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829939.069471:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829939.069476:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829939.069481:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069485:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829939.069486:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069487:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.069488:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829939.069490:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069494:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829939.069497:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069501:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829939.069503:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069503:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.069504:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069505:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829939.069507:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829939.069508:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829939.069510:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829939.069512:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829939.069514:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829939.069519:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327394176:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829939.069522:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069523:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.069525:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115a40 x1874671327394176/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069528:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829939.069530:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829939.069531:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829939.069533:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829939.069534:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829939.069535:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829939.069536:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829939.069538:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829939.069540:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829939.069541:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829939.069542:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115a40.
00000100:00000001:0.0:1787829939.069544:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829939.069544:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069546:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069551:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829939.069552:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069553:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829939.069554:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829939.069555:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829939.069557:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069561:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829939.069562:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069563:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829939.069565:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829939.069566:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829939.069567:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394304:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829939.069569:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829939.069571:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829939.069572:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069573:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829939.069574:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829939.069576:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829939.069578:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829939.069579:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829939.069581:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829939.069583:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394304, portal 10
00000100:00000001:0.0:1787829939.069584:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829939.069585:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402443840 : -116696307107776 : ffff95dd86117840)
00000100:00000040:0.0:1787829939.069589:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069593:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829939.069594:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829939.069596:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829939.069597:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327394304, offset 0
00000400:00000200:0.0:1787829939.069599:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069601:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829939.069603:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829939.069605:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829939.069607:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829939.069608:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829939.069610:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069612:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40852:40852:256:4294967295] 192.168.203.116@tcp LPNI seq info [40852:40852:8:4294967295]
00000400:00000200:0.0:1787829939.069614:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829939.069618:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829939.069621:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069622:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829939.069624:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829939.069627:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829939.069630:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829939.069631:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829939.069634:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069635:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829939.069637:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829939.069639:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829939.069641:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829939.069642:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829939.069643:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829939.069645:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069650:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.069652:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069675:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069676:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829939.069678:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829939.069681:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829939.069683:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829939.069684:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298759179
00000100:00000001:0.0:1787829939.069686:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298759852
00000100:00000001:0.0:1787829939.069688:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298759179
00000100:00000001:0.0:1787829939.069689:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298759852
00000100:00000001:0.0:1787829939.069690:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829939.069691:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069692:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829939.069694:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829939/real 1787829939]  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829939.069698:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829939.069699:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829939.069700:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829939.069702:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829939.069703:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829939.069704:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829939.069706:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829939.069710:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069714:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829939.069715:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069716:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.069716:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829939.069718:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069722:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829939.069724:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069727:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829939.069728:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069729:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.069730:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069731:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829939.069732:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829939.069733:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829939.069735:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829939.069736:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829939.069738:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829939.069742:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394304:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829939.069745:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829939.069747:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069751:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829939.069752:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069753:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829939.069754:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829939.069755:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829939.069756:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394432:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829939.069758:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829939.069759:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829939.069760:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069761:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829939.069763:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829939.069764:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829939.069766:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829939.069767:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829939.069769:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829939.069770:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394432, portal 4
00000100:00000001:0.0:1787829939.069771:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829939.069772:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402438080 : -116696307113536 : ffff95dd861161c0)
00000100:00000040:0.0:1787829939.069775:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069779:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829939.069780:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829939.069781:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829939.069783:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327394432, offset 0
00000400:00000200:0.0:1787829939.069785:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069786:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829939.069788:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829939.069791:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829939.069793:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829939.069795:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829939.069797:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069799:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40853:40853:256:4294967295] 192.168.203.116@tcp LPNI seq info [40853:40853:8:4294967295]
00000400:00000200:0.0:1787829939.069801:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829939.069805:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829939.069808:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069809:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829939.069812:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829939.069813:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829939.069816:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829939.069817:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829939.069819:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069821:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829939.069822:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829939.069824:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829939.069826:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829939.069827:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829939.069828:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829939.069831:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069835:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.069837:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069840:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069841:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829939.069842:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829939.069844:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829939.069846:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829939.069847:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298759186
00000100:00000001:0.0:1787829939.069848:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298759852
00000100:00000001:0.0:1787829939.069850:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298759186
00000100:00000001:0.0:1787829939.069851:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298759852
00000100:00000001:0.0:1787829939.069852:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829939.069854:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.069855:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829939.069857:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829939/real 1787829939]  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829939.069860:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829939.069861:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829939.069862:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829939.069880:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829939.069881:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829939.069882:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829939.069884:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829939.069889:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069892:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829939.069893:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069894:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.069895:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829939.069897:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069900:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829939.069903:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069906:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829939.069908:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069908:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.069909:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.069910:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829939.069911:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829939.069913:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829939.069915:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829939.069916:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829939.069918:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829939.069922:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394432:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829939.069924:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829939.069926:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069929:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829939.069930:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069931:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829939.069932:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829939.069933:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829939.069935:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394560:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829939.069936:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829939.069938:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829939.069938:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829939.069939:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829939.069942:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829939.069943:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829939.069945:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829939.069946:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829939.069948:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829939.069949:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394560, portal 4
00000100:00000001:0.0:1787829939.069951:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829939.069951:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440960 : -116696307110656 : ffff95dd86116d00)
00000100:00000040:0.0:1787829939.069955:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.069959:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829939.069960:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829939.069961:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829939.069963:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327394560, offset 0
00000400:00000200:0.0:1787829939.069965:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069967:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829939.069969:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829939.069971:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829939.069972:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829939.069974:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829939.069975:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829939.069977:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40854:40854:256:4294967295] 192.168.203.116@tcp LPNI seq info [40854:40854:8:4294967295]
00000400:00000200:0.0:1787829939.069979:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829939.069983:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829939.069985:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069987:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829939.069989:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829939.069990:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787829939.069993:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829939.069994:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787829939.069996:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829939.069997:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829939.069999:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829939.070001:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829939.070003:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829939.070004:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829939.070005:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829939.070007:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070011:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.070013:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829964 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070020:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.070021:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829939.070022:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829939.070024:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829939.070026:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829939.070027:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298759191
00000100:00000001:0.0:1787829939.070029:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298759852
00000100:00000001:0.0:1787829939.070030:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298759191
00000100:00000001:0.0:1787829939.070031:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298759852
00000100:00000001:0.0:1787829939.070032:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829939.070033:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.070034:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829939.070036:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829939/real 1787829939]  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829939.070040:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829939.070041:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829939.070041:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829939.070043:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829939.070043:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829939.070044:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829939.070046:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829939.070050:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070054:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829939.070054:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070055:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.070056:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829939.070058:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070062:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829939.070064:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070068:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829939.070069:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070070:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829939.070071:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070071:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829939.070072:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829939.070074:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829939.070075:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829939.070076:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829939.070078:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829939.070082:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394560:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829939.070084:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070085:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.070086:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117840 x1874671327394304/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070090:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829939.070090:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829939.070091:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829939.070093:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829939.070094:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829939.070094:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829939.070096:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829939.070097:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829939.070099:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829939.070099:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829939.070100:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117840.
00000100:00000001:0.0:1787829939.070102:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829939.070103:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070104:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.070106:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861161c0 x1874671327394432/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070109:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829939.070110:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829939.070110:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829939.070112:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829939.070112:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829939.070113:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829939.070114:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829939.070114:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:0.0:1787829939.070115:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829939.070116:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829939.070117:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861161c0.
00000100:00000001:0.0:1787829939.070118:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829939.070119:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070119:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829939.070121:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116d00 x1874671327394560/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829964 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829939.070124:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829939.070125:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829939.070125:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530000.
02000000:00000001:0.0:1787829939.070126:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829939.070127:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829939.070127:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829939.070128:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829939.070129:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:0.0:1787829939.070130:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829939.070131:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829939.070131:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116d00.
00000100:00000001:0.0:1787829939.070132:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829939.070133:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070134:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.070136:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829939.070137:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.070138:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829939.070139:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829939.580757:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829939.580763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829939.580765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829939.580769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829939.580772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829939.580774:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.580776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829939.580778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829939.580780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829939.772743:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298759852, at 4298759872
00000100:00000040:3.0:1787829939.772743:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298759852, at 4298759872
00000100:00000010:3.0:1787829939.772756:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc300.
00000100:00000010:2.0:1787829939.772757:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc480.
00010000:00000001:0.0:1787829939.900761:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829939.900766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829939.900771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829939.900772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829939.900775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829939.900776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829939.900817:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829939.900819:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829939.900822:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829940.604748:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829940.604755:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829940.604758:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829940.604763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829940.604769:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829940.604772:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829940.604774:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829940.604777:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829940.604780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829940.924724:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829940.924729:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829940.924732:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829940.924733:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829940.924735:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829940.924737:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829940.924739:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829940.924741:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829940.924743:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829940.924744:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829940.924745:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829940.924747:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829940.924781:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829940.924784:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829940.924787:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829941.628783:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829941.628791:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829941.628792:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829941.628796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829941.628799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829941.628802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829941.628804:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829941.628806:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829941.628808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829941.948758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829941.948764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829941.948770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829941.948772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829941.948775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829941.948777:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829941.948824:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829941.948833:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829941.948837:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829942.652768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829942.652774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829942.652775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829942.652779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829942.652781:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829942.652784:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829942.652786:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829942.652787:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829942.652789:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829942.972750:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829942.972758:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829942.972763:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829942.972764:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829942.972767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829942.972768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829942.972819:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829942.972822:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829942.972825:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829943.676752:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829943.676758:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829943.676760:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829943.676763:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829943.676766:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829943.676768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829943.676770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829943.676771:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829943.676774:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829943.996746:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829943.996751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829943.996755:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829943.996756:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829943.996758:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829943.996759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829943.996816:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829943.996819:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829943.996822:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829944.188743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829944.188753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829944.188756:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829944.188759:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829944.188763:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829944.188770:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4091
00000400:00000200:2.0:1787829944.188779:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829944.188784:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829944.188788:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829944.188790:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.188793:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829944.188795:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.188797:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829944.188801:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829944.188803:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829944.188806:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829944.188809:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829944.188811:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829944.188813:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829944.188816:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.188818:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829944.188820:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.188821:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829944.188824:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829944.188837:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829944.188840:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829944.188843:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829944.188846:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829944.188849:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.188851:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829944.188853:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829944.188861:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861170c0.
00000020:00000040:2.0:1787829944.188865:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829944.188868:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829944.188870:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.188873:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829944.188875:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829944.188883:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829944.188890:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000100:00000001:2.0:1787829944.188896:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829944.188904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829944.188914:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829944.188917:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829944.188921:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048282787] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829944.188942:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829944.188946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829944.188949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829944.188950:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829944.188951:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829944.188954:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829944.188956:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4091
00000400:00000200:2.0:1787829944.188959:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829944.188963:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829944.188965:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829944.188969:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.188971:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829944.188974:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829944.188977:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829944.188977:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829944.188981:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787829944.188982:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787829944.188984:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1787829944.188986:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829944.188990:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1787829944.188991:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787829944.188994:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829944.188999:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829944.189004:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:0.0:1787829944.189005:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829944.189008:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829944.189011:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829944.189013:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787829944.189015:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1787829944.189016:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787829944.189019:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
02000000:00000001:0.0:1787829944.189020:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.189022:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00100000:0.0:1787829944.189025:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327394688:192.168.203.116@tcp:250:kworker.0
00000020:00000040:2.0:1787829944.189027:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829944.189032:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787829944.189033:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:0.0:1787829944.189036:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787829944.189036:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.189039:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829944.189040:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829944.189043:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787829944.189046:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1787829944.189048:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:2.0:1787829944.189049:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.189051:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.189052:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829944.189057:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000008:00100000:2.0:1787829944.189057:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829944.189062:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829944.189063:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00000001:2.0:1787829944.189067:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829944.189071:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000010:2.0:1787829944.189072:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114b40.
00000020:00000040:2.0:1787829944.189075:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000200:0.0:1787829944.189077:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394688, portal 25
02000000:00000001:2.0:1787829944.189077:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829944.189080:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189081:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829944.189083:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402441920 : -116696307109696 : ffff95dd861170c0)
00000100:00000001:2.0:1787829944.189083:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829944.189085:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829944.189088:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829944.189090:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787829944.189092:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829944.189097:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189103:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:2.0:1787829944.189107:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1787829944.189111:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829944.189117:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000001:2.0:1787829944.189117:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829944.189121:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829944.189124:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327394688, offset 0
00000100:00000040:2.0:1787829944.189125:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2087935] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829944.189134:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829944.189135:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829944.189138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829944.189141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829944.189143:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829944.189144:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787829944.189146:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829944.189147:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829944.189151:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4091
00000400:00000200:0.0:1787829944.189153:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:2.0:1787829944.189155:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.189159:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787829944.189162:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829944.189165:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829944.189169:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1787829944.189171:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829944.189172:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829944.189178:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40855:40855:256:4294967295] 192.168.203.116@tcp LPNI seq info [40855:40855:8:4294967295]
00000100:00000040:2.0:1787829944.189178:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189184:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829944.189185:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829944.189189:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189191:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829944.189195:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829944.189198:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000400:00000200:0.0:1787829944.189199:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1787829944.189200:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829944.189204:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829944.189206:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000200:0.0:1787829944.189208:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:2.0:1787829944.189209:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:0.0:1787829944.189214:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00000040:2.0:1787829944.189214:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189219:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829944.189220:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189221:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000800:00000200:0.0:1787829944.189225:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787829944.189225:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:0.0:1787829944.189230:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000100:00080000:2.0:1787829944.189231:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189236:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829944.189238:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787829944.189241:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000001:2.0:1787829944.189242:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829944.189244:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.189245:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829944.189247:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829944.189248:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.189249:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829944.189252:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117480.
00000020:00000040:2.0:1787829944.189256:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:0.1:1787829944.189257:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
02000000:00000001:2.0:1787829944.189259:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829944.189262:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1787829944.189265:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189266:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:0.0:1787829944.189268:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
02000000:00000001:2.0:1787829944.189269:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829944.189272:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829944.189275:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:2.0:1787829944.189276:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000800.
00000400:00000200:0.0:1787829944.189280:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829944.189280:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829944.189286:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00100000:2.0:1787829944.189288:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1787829944.189289:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829944.189295:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829944.189301:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829944.189306:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829944.189309:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829944.189315:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc873e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189321:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787829944.189322:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829944.189325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787829944.189329:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829944.189329:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829944.189333:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829944.189336:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829944.189343:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829944.189347:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829944.189348:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4091
00000100:00000001:0.0:1787829944.189351:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829944.189357:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829944.189362:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.189365:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829944.189368:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298764972
00000400:00000200:2.0:1787829944.189370:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000010:0.0:1787829944.189373:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fccf00.
00000100:00080000:2.0:1787829944.189374:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787829944.189379:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787829944.189380:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189383:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829944.189384:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189385:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829944.189387:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829944.189388:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829944.189391:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829944.189393:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829944.189394:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298764972
00000020:00000001:2.0:1787829944.189394:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829944.189399:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000010:0.0:1787829944.189401:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc4e0.
00000100:00000040:2.0:1787829944.189405:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829944.189407:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787829944.189409:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829944.189414:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829944.189416:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829944.189419:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1787829944.189420:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829944.189425:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829944.189427:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829944.189429:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787829944.189433:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829944.189437:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829944/real 1787829944]  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829944.189438:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829944.189443:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000001:0.0:1787829944.189446:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829944.189448:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1787829944.189448:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829944.189451:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000100:00000001:2.0:1787829944.189453:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829944.189455:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000008:00100000:2.0:1787829944.189457:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1787829944.189458:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829944.189464:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000008:00000001:2.0:1787829944.189464:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.189469:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829944.189470:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787829944.189475:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114f00.
00000020:00000040:2.0:1787829944.189480:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000200:0.0:1787829944.189485:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787829944.189486:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829944.189489:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829944.189494:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829944.189497:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829944.189499:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1787829944.189501:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189502:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:2.0:1787829944.189506:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:0.0:1787829944.189507:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.189510:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829944.189512:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829944.189517:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787829944.189522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829944.189536:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829944.189542:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1787829944.189543:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829944.189547:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189554:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1787829944.189555:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbc790c9] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189558:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189562:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.189564:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189568:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787829944.189572:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189573:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829944.189580:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:2.0:1787829944.189582:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4101)
00000100:00080000:0.0:1787829944.189586:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829944.189590:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829944.189596:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829944.189604:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327394688:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829944.189610:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189611:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.189615:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861170c0 x1874671327394688/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189621:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829944.189623:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829944.189626:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829944.189628:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829944.189630:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829944.189631:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829944.189633:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829944.189635:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:0.0:1787829944.189638:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829944.189640:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829944.189642:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861170c0.
00000100:00000001:0.0:1787829944.189645:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829944.189646:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189648:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189653:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829944.189674:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189677:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829944.189678:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829944.189679:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829944.189688:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189693:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829944.189694:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.189695:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829944.189697:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829944.189698:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829944.189699:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394816:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829944.189702:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829944.189703:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829944.189704:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.189705:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829944.189707:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829944.189708:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829944.189711:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829944.189712:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829944.189714:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829944.189716:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394816, portal 10
00000100:00000001:0.0:1787829944.189718:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829944.189718:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402432320 : -116696307119296 : ffff95dd86114b40)
00000100:00000040:0.0:1787829944.189723:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189727:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829944.189729:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829944.189730:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829944.189732:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327394816, offset 0
00000400:00000200:0.0:1787829944.189734:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829944.189737:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.189739:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829944.189741:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829944.189744:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829944.189746:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829944.189748:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829944.189751:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40856:40856:256:4294967295] 192.168.203.116@tcp LPNI seq info [40856:40856:8:4294967295]
00000400:00000200:0.0:1787829944.189755:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.189761:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829944.189766:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829944.189769:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829944.189773:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829944.189775:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829944.189780:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829944.189782:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829944.189786:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829944.189788:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829944.189790:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829944.189794:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829944.189796:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829944.189798:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829944.189800:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829944.189804:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189809:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.189811:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189821:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189823:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829944.189826:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829944.189944:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829944.189946:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829944.189947:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298764299
00000100:00000001:0.0:1787829944.189949:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298764972
00000100:00000001:0.0:1787829944.189951:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298764299
00000100:00000001:0.0:1787829944.189952:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298764972
00000100:00000001:0.0:1787829944.189953:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829944.189955:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.189956:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829944.189959:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829944/real 1787829944]  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829944.189963:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829944.189965:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829944.189966:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829944.189968:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829944.189969:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829944.189970:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829944.189972:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829944.189977:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189981:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829944.189981:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189982:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.189983:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829944.189985:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189989:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829944.189991:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.189995:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829944.189997:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.189998:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.189999:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190000:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829944.190001:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829944.190002:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829944.190004:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829944.190005:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829944.190007:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829944.190011:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394816:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829944.190013:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829944.190015:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190019:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829944.190020:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.190021:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829944.190022:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829944.190023:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829944.190025:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394944:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829944.190027:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829944.190028:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829944.190029:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.190030:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829944.190032:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829944.190034:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829944.190035:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829944.190037:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829944.190039:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829944.190041:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327394944, portal 4
00000100:00000001:0.0:1787829944.190042:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829944.190043:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402442880 : -116696307108736 : ffff95dd86117480)
00000100:00000040:0.0:1787829944.190047:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190053:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829944.190055:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829944.190057:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829944.190060:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327394944, offset 0
00000400:00000200:0.0:1787829944.190064:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829944.190067:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.190070:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829944.190073:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829944.190076:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829944.190078:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829944.190080:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829944.190083:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40857:40857:256:4294967295] 192.168.203.116@tcp LPNI seq info [40857:40857:8:4294967295]
00000400:00000200:0.0:1787829944.190087:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.190093:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829944.190096:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829944.190099:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829944.190103:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829944.190106:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829944.190110:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829944.190112:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829944.190115:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829944.190117:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829944.190120:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829944.190123:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829944.190126:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829944.190128:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829944.190130:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829944.190134:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190142:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.190145:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190149:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.190150:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829944.190153:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829944.190155:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829944.190157:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829944.190158:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298764306
00000100:00000001:0.0:1787829944.190160:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298764972
00000100:00000001:0.0:1787829944.190161:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298764306
00000100:00000001:0.0:1787829944.190162:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298764972
00000100:00000001:0.0:1787829944.190163:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829944.190164:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.190165:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829944.190168:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829944/real 1787829944]  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829944.190172:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829944.190173:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829944.190174:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829944.190175:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829944.190176:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829944.190178:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829944.190179:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829944.190184:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190191:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829944.190192:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190193:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.190194:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829944.190196:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190199:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829944.190201:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190205:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829944.190206:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190207:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.190207:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190208:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829944.190209:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829944.190211:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829944.190212:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829944.190213:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829944.190215:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829944.190219:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327394944:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829944.190221:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829944.190223:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190227:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829944.190228:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.190229:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829944.190230:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829944.190232:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829944.190233:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395072:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829944.190235:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829944.190236:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829944.190237:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829944.190238:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829944.190240:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829944.190241:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829944.190243:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829944.190244:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829944.190246:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829944.190248:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395072, portal 4
00000100:00000001:0.0:1787829944.190249:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829944.190250:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402433280 : -116696307118336 : ffff95dd86114f00)
00000100:00000040:0.0:1787829944.190253:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190258:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829944.190260:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829944.190262:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829944.190264:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327395072, offset 0
00000400:00000200:0.0:1787829944.190268:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829944.190271:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.190274:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829944.190277:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829944.190279:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829944.190281:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829944.190283:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829944.190286:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40858:40858:256:4294967295] 192.168.203.116@tcp LPNI seq info [40858:40858:8:4294967295]
00000400:00000200:0.0:1787829944.190289:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829944.190295:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829944.190300:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829944.190303:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829944.190306:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829944.190309:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787829944.190313:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829944.190316:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787829944.190319:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829944.190320:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829944.190323:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829944.190326:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829944.190328:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829944.190329:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829944.190331:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829944.190333:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190338:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.190340:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190343:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.190344:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829944.190346:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829944.190349:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829944.190352:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829944.190354:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298764311
00000100:00000001:0.0:1787829944.190356:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298764972
00000100:00000001:0.0:1787829944.190358:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298764311
00000100:00000001:0.0:1787829944.190360:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298764972
00000100:00000001:0.0:1787829944.190362:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829944.190364:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.190366:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829944.190369:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829944/real 1787829944]  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829944.190376:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829944.190378:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829944.190379:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829944.190382:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829944.190383:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829944.190385:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829944.190388:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829944.190396:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190402:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829944.190404:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190405:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.190407:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829944.190410:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190416:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829944.190419:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190423:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829944.190424:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190425:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829944.190426:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190427:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829944.190428:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829944.190430:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829944.190432:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829944.190433:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829944.190435:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829944.190440:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395072:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829944.190442:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190443:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.190446:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114b40 x1874671327394816/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190452:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829944.190453:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829944.190455:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829944.190458:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829944.190459:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829944.190460:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829944.190462:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829944.190464:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:0.0:1787829944.190466:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829944.190467:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829944.190469:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114b40.
00000100:00000001:0.0:1787829944.190471:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829944.190472:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190473:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.190476:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117480 x1874671327394944/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190483:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829944.190484:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829944.190485:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531800.
02000000:00000001:0.0:1787829944.190487:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829944.190488:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829944.190489:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829944.190491:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829944.190493:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000800.
02000000:00000001:0.0:1787829944.190495:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829944.190497:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829944.190498:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117480.
00000100:00000001:0.0:1787829944.190500:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829944.190501:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190503:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829944.190506:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114f00 x1874671327395072/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829944.190512:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829944.190513:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829944.190515:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533400.
02000000:00000001:0.0:1787829944.190517:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829944.190518:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829944.190518:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829944.190519:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829944.190520:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:0.0:1787829944.190522:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829944.190522:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829944.190523:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114f00.
00000100:00000001:0.0:1787829944.190525:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829944.190525:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190527:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.190529:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829944.190530:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.190531:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829944.190532:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829944.700749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829944.700756:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829944.700757:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829944.700761:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829944.700763:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829944.700766:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.700767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829944.700769:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829944.700771:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829944.892754:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298764972, at 4298764992
00000100:00000040:3.0:1787829944.892759:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298764972, at 4298764992
00000100:00000010:2.0:1787829944.892766:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fccf00.
00000100:00000010:3.0:1787829944.892768:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc4e0.
00010000:00000001:0.0:1787829945.020711:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829945.020715:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829945.020719:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829945.020720:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829945.020722:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829945.020722:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829945.020760:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829945.020762:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829945.020764:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829945.724744:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829945.724752:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829945.724755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829945.724760:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829945.724764:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829945.724768:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829945.724770:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829945.724773:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829945.724776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044753:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829946.044761:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829946.044763:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829946.044767:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044772:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829946.044774:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044776:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044778:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829946.044784:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829946.044785:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829946.044786:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044788:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829946.044789:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044791:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044792:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829946.044796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829946.044797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829946.044859:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829946.044862:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829946.044864:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829946.748773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829946.748779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829946.748781:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829946.748785:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829946.748789:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829946.748793:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829946.748796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829946.748798:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829946.748800:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829947.068790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829947.068796:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829947.068803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829947.068804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829947.068820:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829947.068822:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829947.068878:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829947.068881:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829947.068884:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829947.772758:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829947.772763:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829947.772765:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829947.772769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829947.772771:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829947.772775:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829947.772776:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829947.772778:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829947.772780:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829948.092781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829948.092789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829948.092795:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829948.092797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829948.092811:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829948.092812:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829948.092873:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829948.092877:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829948.092880:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829948.796829:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829948.796840:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829948.796842:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829948.796848:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829948.796851:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829948.796875:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829948.796878:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829948.796880:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829948.796883:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829949.116790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829949.116813:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829949.116821:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829949.116823:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829949.116827:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829949.116828:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829949.116901:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829949.116905:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829949.116910:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829949.308783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829949.308808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829949.308811:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829949.308816:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829949.308820:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829949.308828:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4096
00000400:00000200:1.0:1787829949.308838:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829949.308844:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829949.308848:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829949.308850:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.308853:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829949.308856:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.308858:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829949.308878:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829949.308880:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829949.308883:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829949.308887:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829949.308889:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829949.308890:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829949.308893:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.308894:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829949.308896:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.308897:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829949.308900:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829949.308903:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829949.308906:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829949.308909:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829949.308913:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829949.308915:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.308918:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829949.308920:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829949.308947:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0b40.
00000020:00000040:1.0:1787829949.308950:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829949.308954:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829949.308956:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.308959:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829949.308962:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829949.308967:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829949.308975:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7400.
00000100:00000001:1.0:1787829949.308982:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829949.309008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829949.309018:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829949.309023:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829949.309028:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000370807da] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829949.309051:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829949.309054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829949.309056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829949.309058:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829949.309059:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829949.309061:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829949.309063:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4096
00000400:00000200:1.0:1787829949.309066:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829949.309068:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829949.309070:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829949.309072:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309074:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829949.309075:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309077:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829949.309079:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829949.309080:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829949.309082:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829949.309084:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829949.309086:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829949.309104:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829949.309107:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309109:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829949.309111:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309112:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829949.309114:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:0.0:1787829949.309116:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787829949.309117:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309122:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309123:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:1.0:1787829949.309124:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829949.309127:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1787829949.309131:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829949.309136:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829949.309137:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829949.309138:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829949.309144:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829949.309150:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309152:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829949.309153:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309155:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829949.309157:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0f00.
02000000:00000001:0.0:1787829949.309160:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:1.0:1787829949.309161:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:0.0:1787829949.309163:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829949.309165:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829949.309168:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829949.309168:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.309173:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:0.0:1787829949.309176:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327395200:192.168.203.116@tcp:250:kworker.0
02000000:00000001:1.0:1787829949.309176:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829949.309179:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309186:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:1.0:1787829949.309186:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829949.309191:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787829949.309191:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.309195:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.309198:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:1.0:1787829949.309202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:0.0:1787829949.309204:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829949.309208:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829949.309218:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829949.309218:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829949.309223:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787829949.309224:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829949.309233:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000040:1.0:1787829949.309234:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d265f843] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829949.309247:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1787829949.309249:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395200, portal 25
00000100:00080000:1.0:1787829949.309251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1787829949.309254:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829949.309256:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418996544 : -116696290555072 : ffff95dd870e0b40)
00000100:00080000:1.0:1787829949.309259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829949.309261:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1787829949.309266:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787829949.309266:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829949.309274:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829949.309278:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1787829949.309281:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4096
00000100:00000040:0.0:1787829949.309284:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829949.309288:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000400:00000200:1.0:1787829949.309292:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829949.309293:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327395200, offset 0
00000400:00000200:1.0:1787829949.309301:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000400:00000200:0.0:1787829949.309306:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:1.0:1787829949.309308:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829949.309314:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829949.309316:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829949.309321:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829949.309327:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000040:1.0:1787829949.309327:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309333:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829949.309336:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1787829949.309337:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1787829949.309339:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829949.309342:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1787829949.309344:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000040:1.0:1787829949.309344:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829949.309348:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829949.309349:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000020:00000001:1.0:1787829949.309351:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:0.0:1787829949.309354:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:1.0:1787829949.309357:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309360:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1787829949.309362:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40859:40859:256:4294967295] 192.168.203.116@tcp LPNI seq info [40859:40859:8:4294967295]
00000100:00000040:1.0:1787829949.309365:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309367:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829949.309369:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829949.309375:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829949.309377:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829949.309379:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829949.309382:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829949.309385:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829949.309388:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.309392:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829949.309393:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000008:00100000:1.0:1787829949.309395:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829949.309399:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829949.309403:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309403:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829949.309407:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e2580.
00000800:00000010:0.0:1787829949.309410:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000020:00000040:1.0:1787829949.309411:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829949.309416:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829949.309417:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.309419:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829949.309421:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829949.309422:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1787829949.309426:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
02000000:00000010:1.0:1787829949.309429:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7800.
00000800:00000200:0.0:1787829949.309431:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000100:00000001:1.0:1787829949.309433:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787829949.309439:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00100000:1.0:1787829949.309441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1787829949.309444:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829949.309455:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309455:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000010:0.0:1787829949.309459:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000100:00000001:1.0:1787829949.309460:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829949.309466:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e588eb2] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787829949.309468:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829949.309478:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787829949.309478:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829949.309484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829949.309487:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000100:00080000:1.0:1787829949.309487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:0.0:1787829949.309491:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:1.0:1787829949.309491:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829949.309495:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829949.309497:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787829949.309498:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:1.0:1787829949.309501:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4096
00000100:00000200:0.0:1787829949.309505:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829949.309507:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829949.309512:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829949.309514:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829949.309518:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309522:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829949.309526:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829949.309528:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787829949.309529:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000040:0.0:1787829949.309534:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787829949.309534:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829949.309538:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829949.309543:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829949.309546:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:0.0:1787829949.309547:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309553:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:1.0:1787829949.309553:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829949.309559:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829949.309560:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829949.309569:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:0.0:1787829949.309573:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829949.309574:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787829949.309577:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298770092
00000100:00000040:1.0:1787829949.309580:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:0.0:1787829949.309584:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc7e0.
00000100:00000001:1.0:1787829949.309585:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829949.309590:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:0.0:1787829949.309591:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:1.0:1787829949.309594:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829949.309598:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309599:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298770092
02000000:00000001:1.0:1787829949.309600:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:0.0:1787829949.309606:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc0c0.
02000000:00000001:1.0:1787829949.309606:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829949.309611:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
02000000:00000001:1.0:1787829949.309611:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.309616:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829949.309618:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829949.309620:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829949.309621:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309622:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:1.0:1787829949.309625:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829949.309626:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787829949.309629:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829949.309631:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1787829949.309633:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829949.309635:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829949.309638:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829949/real 1787829949]  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829949.309638:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829949.309641:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829949.309644:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829949.309650:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829949.309654:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829949.309659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829949.309666:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829949.309669:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1787829949.309694:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829949.309697:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829949.309700:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829949.309705:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829949.309707:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829949.309710:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829949.309717:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787829949.309718:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3020206] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829949.309747:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829949.309753:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4107)
00000100:00000200:0.0:1787829949.309754:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309762:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829949.309764:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.309766:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.309768:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829949.309771:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309777:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829949.309782:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309804:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829949.309807:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.309808:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.309809:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.309811:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829949.309813:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829949.309815:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829949.309818:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829949.309821:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829949.309824:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829949.309830:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327395200:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829949.309834:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.309836:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.309837:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0b40 x1874671327395200/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309842:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829949.309844:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829949.309846:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829949.309849:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829949.309850:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829949.309852:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829949.309854:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829949.309856:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7400.
02000000:00000001:0.0:1787829949.309859:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829949.309861:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829949.309862:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0b40.
00000100:00000001:0.0:1787829949.309865:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829949.309866:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.309868:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.309874:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829949.309876:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.309877:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829949.309879:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829949.309880:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829949.309883:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309888:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829949.309889:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.309890:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829949.309892:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829949.309893:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829949.309895:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395328:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829949.309898:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829949.309899:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829949.309900:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.309902:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829949.309904:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829949.309906:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829949.309908:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829949.309910:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829949.309913:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829949.309915:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395328, portal 10
00000100:00000001:0.0:1787829949.309917:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829949.309918:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418997504 : -116696290554112 : ffff95dd870e0f00)
00000100:00000040:0.0:1787829949.309923:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309928:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829949.309930:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829949.309931:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829949.309933:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327395328, offset 0
00000400:00000200:0.0:1787829949.309936:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829949.309939:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829949.309941:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829949.309943:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829949.309945:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829949.309947:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829949.309949:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829949.309951:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40860:40860:256:4294967295] 192.168.203.116@tcp LPNI seq info [40860:40860:8:4294967295]
00000400:00000200:0.0:1787829949.309954:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829949.309959:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829949.309962:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829949.309964:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829949.309967:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829949.309969:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829949.309973:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829949.309974:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829949.309977:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829949.309978:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829949.309980:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829949.309982:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829949.309984:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829949.309986:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829949.309988:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829949.309990:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.309995:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.309997:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310002:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310003:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829949.310005:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829949.310008:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829949.310010:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829949.310012:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298769419
00000100:00000001:0.0:1787829949.310013:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298770092
00000100:00000001:0.0:1787829949.310015:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298769419
00000100:00000001:0.0:1787829949.310018:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298770092
00000100:00000001:0.0:1787829949.310020:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829949.310022:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310023:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829949.310025:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829949/real 1787829949]  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829949.310030:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829949.310031:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829949.310032:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829949.310034:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829949.310035:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829949.310036:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829949.310038:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829949.310043:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310047:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829949.310048:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310049:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.310050:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829949.310052:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310056:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829949.310058:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310063:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829949.310065:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310066:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.310067:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310069:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829949.310070:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829949.310072:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829949.310074:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829949.310076:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829949.310078:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829949.310083:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395328:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829949.310085:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829949.310087:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310091:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829949.310092:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.310094:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829949.310095:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829949.310097:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829949.310099:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395456:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829949.310101:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829949.310102:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829949.310103:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.310104:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829949.310106:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829949.310107:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829949.310109:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829949.310111:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829949.310113:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829949.310115:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395456, portal 4
00000100:00000001:0.0:1787829949.310116:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829949.310117:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419003264 : -116696290548352 : ffff95dd870e2580)
00000100:00000040:0.0:1787829949.310121:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310126:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829949.310128:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829949.310129:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829949.310131:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327395456, offset 0
00000400:00000200:0.0:1787829949.310133:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829949.310135:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829949.310137:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829949.310138:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829949.310140:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829949.310141:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829949.310143:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829949.310145:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40861:40861:256:4294967295] 192.168.203.116@tcp LPNI seq info [40861:40861:8:4294967295]
00000400:00000200:0.0:1787829949.310148:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829949.310152:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829949.310155:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829949.310157:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829949.310160:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829949.310162:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829949.310164:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829949.310166:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829949.310168:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829949.310170:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829949.310172:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829949.310174:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829949.310176:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829949.310177:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829949.310179:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829949.310181:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310186:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.310188:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310197:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310198:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829949.310199:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829949.310201:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829949.310203:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829949.310205:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298769426
00000100:00000001:0.0:1787829949.310206:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298770092
00000100:00000001:0.0:1787829949.310208:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298769426
00000100:00000001:0.0:1787829949.310209:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298770092
00000100:00000001:0.0:1787829949.310210:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829949.310212:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310213:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829949.310215:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829949/real 1787829949]  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829949.310219:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829949.310220:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829949.310221:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829949.310222:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829949.310224:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829949.310225:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829949.310227:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829949.310233:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310236:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829949.310237:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310238:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.310239:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829949.310241:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310247:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829949.310249:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310254:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829949.310255:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310256:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.310257:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310258:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829949.310259:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829949.310260:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829949.310262:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829949.310263:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829949.310265:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829949.310270:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395456:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829949.310272:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829949.310274:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310278:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829949.310279:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.310280:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829949.310281:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829949.310282:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829949.310284:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395584:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829949.310286:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829949.310287:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829949.310289:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829949.310290:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829949.310292:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829949.310293:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829949.310296:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829949.310298:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829949.310299:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829949.310301:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395584, portal 4
00000100:00000001:0.0:1787829949.310302:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829949.310304:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418999424 : -116696290552192 : ffff95dd870e1680)
00000100:00000040:0.0:1787829949.310308:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310312:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829949.310314:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829949.310315:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000200:0.0:1787829949.310316:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327395584, offset 0
00000400:00000200:0.0:1787829949.310318:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829949.310320:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829949.310322:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829949.310324:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829949.310325:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829949.310327:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829949.310328:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829949.310330:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40862:40862:256:4294967295] 192.168.203.116@tcp LPNI seq info [40862:40862:8:4294967295]
00000400:00000200:0.0:1787829949.310333:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829949.310337:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829949.310340:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829949.310341:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000800:00000200:0.0:1787829949.310344:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829949.310345:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787829949.310348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829949.310350:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787829949.310352:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829949.310354:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:0.0:1787829949.310356:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829949.310358:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829949.310360:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:0.0:1787829949.310362:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:0.0:1787829949.310363:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829949.310365:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310369:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.310372:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310376:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310377:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829949.310379:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829949.310380:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829949.310382:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829949.310383:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298769431
00000100:00000001:0.0:1787829949.310385:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298770092
00000100:00000001:0.0:1787829949.310387:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298769431
00000100:00000001:0.0:1787829949.310388:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298770092
00000100:00000001:0.0:1787829949.310389:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829949.310391:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310392:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829949.310394:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829949/real 1787829949]  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829949.310398:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829949.310399:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829949.310400:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829949.310401:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829949.310402:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829949.310404:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829949.310406:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829949.310412:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310416:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829949.310417:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310418:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.310419:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829949.310421:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310425:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829949.310427:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310431:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829949.310432:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310433:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829949.310434:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310435:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829949.310436:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829949.310437:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829949.310439:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829949.310440:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829949.310443:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829949.310447:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395584:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829949.310449:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310451:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.310453:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0f00 x1874671327395328/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310458:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829949.310459:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829949.310460:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829949.310461:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829949.310462:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829949.310463:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829949.310464:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829949.310466:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6800.
02000000:00000001:0.0:1787829949.310467:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829949.310468:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829949.310469:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0f00.
00000100:00000001:0.0:1787829949.310471:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829949.310472:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310474:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.310476:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e2580 x1874671327395456/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310480:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829949.310481:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829949.310482:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533800.
02000000:00000001:0.0:1787829949.310483:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829949.310484:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829949.310485:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829949.310486:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829949.310487:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7800.
02000000:00000001:0.0:1787829949.310488:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829949.310488:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829949.310489:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e2580.
00000100:00000001:0.0:1787829949.310491:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829949.310491:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310492:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829949.310494:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e1680 x1874671327395584/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829949.310498:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829949.310499:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829949.310500:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533c00.
02000000:00000001:0.0:1787829949.310501:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829949.310502:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829949.310503:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829949.310504:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829949.310505:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829949.310507:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829949.310507:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829949.310508:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e1680.
00000100:00000001:0.0:1787829949.310509:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829949.310510:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310511:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.310513:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829949.310514:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.310515:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829949.310516:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829949.820766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829949.820773:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829949.820776:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829949.820782:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829949.820786:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829949.820802:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.820806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829949.820810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829949.820813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829950.012809:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298770092, at 4298770112
00000100:00000040:3.0:1787829950.012811:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298770092, at 4298770112
00000100:00000010:2.0:1787829950.012829:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc7e0.
00000100:00000010:3.0:1787829950.012835:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00010000:00000001:0.0:1787829950.140773:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829950.140778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829950.140796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829950.140798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829950.140800:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829950.140801:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829950.140861:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829950.140865:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829950.140868:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829950.844777:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829950.844800:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829950.844801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829950.844805:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829950.844808:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829950.844811:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829950.844814:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829950.844816:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829950.844818:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829951.164823:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829951.164831:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829951.164839:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829951.164841:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829951.164846:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829951.164850:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829951.164853:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829951.164856:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829951.164859:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829951.164862:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829951.164864:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829951.164867:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829951.164906:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829951.164909:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829951.164912:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829951.868830:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829951.868840:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829951.868842:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829951.868850:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829951.868854:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829951.868859:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829951.868862:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829951.868866:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829951.868869:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829952.188790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829952.188798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829952.188804:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829952.188806:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829952.188809:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829952.188810:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829952.188867:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829952.188871:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829952.188875:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829952.892799:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829952.892807:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829952.892810:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829952.892815:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829952.892819:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829952.892824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829952.892827:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829952.892829:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829952.892833:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829953.212709:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829953.212713:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829953.212719:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829953.212720:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829953.212722:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829953.212723:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829953.212795:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829953.212800:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829953.212804:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829953.917043:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829953.917051:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829953.917053:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829953.917059:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829953.917062:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829953.917066:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829953.917069:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829953.917071:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829953.917074:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829954.236810:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829954.236819:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829954.236825:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829954.236827:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829954.236830:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829954.236832:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829954.236863:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829954.236867:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829954.236870:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829954.428793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829954.428800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829954.428802:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829954.428805:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829954.428808:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829954.428814:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4102
00000400:00000200:2.0:1787829954.428822:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829954.428828:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829954.428832:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829954.428835:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428838:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829954.428840:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829954.428846:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829954.428848:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829954.428850:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829954.428854:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829954.428863:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829954.428865:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829954.428868:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428870:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829954.428872:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428873:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829954.428876:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829954.428878:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428879:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.428881:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829954.428884:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829954.428885:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.428887:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829954.428889:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829954.428896:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116580.
00000020:00000040:2.0:1787829954.428899:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829954.428902:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829954.428903:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.428906:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829954.428908:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829954.428911:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829954.428916:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003400.
00000100:00000001:2.0:1787829954.428920:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829954.428927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829954.428933:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829954.428936:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829954.428939:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006044e51f] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829954.428958:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829954.428960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829954.428962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829954.428963:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829954.428964:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829954.428966:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829954.428967:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4102
00000400:00000200:2.0:1787829954.428970:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829954.428972:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829954.428973:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829954.428975:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428977:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829954.428979:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428981:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829954.428983:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829954.428984:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829954.428986:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829954.428988:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829954.428990:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829954.428992:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829954.428996:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.428998:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829954.429000:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429003:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829954.429005:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829954.429008:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829954.429012:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787829954.429012:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.429014:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829954.429019:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787829954.429029:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829954.429034:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429035:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787829954.429036:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829954.429042:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000040:0.0:1787829954.429046:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787829954.429046:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429051:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829954.429058:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115680.
00000100:00000001:0.0:1787829954.429060:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787829954.429062:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000100:00000001:0.0:1787829954.429063:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.429069:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829954.429070:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829954.429072:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429074:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829954.429076:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829954.429079:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829954.429081:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.429082:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829954.429088:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327395712:192.168.203.116@tcp:250:kworker.0
02000000:00000010:2.0:1787829954.429089:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829954.429094:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829954.429100:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829954.429102:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787829954.429107:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829954.429109:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787829954.429110:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429113:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829954.429115:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f86ac55] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787829954.429118:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829954.429125:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530800.
00000100:00000001:2.0:1787829954.429128:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429129:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829954.429133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787829954.429138:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:2.0:1787829954.429140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829954.429144:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1787829954.429145:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000100:00080000:2.0:1787829954.429146:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829954.429151:0:36:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1787829954.429153:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00080000:2.0:1787829954.429155:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4102
00000400:00000200:2.0:1787829954.429162:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000200:0.0:1787829954.429163:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395712, portal 25
00000400:00000200:2.0:1787829954.429167:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:0.0:1787829954.429168:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829954.429172:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402439040 : -116696307112576 : ffff95dd86116580)
00000100:00080000:2.0:1787829954.429172:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829954.429179:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:0.0:1787829954.429184:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829954.429184:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829954.429190:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429196:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829954.429200:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829954.429202:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829954.429203:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829954.429205:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829954.429209:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000001:0.0:1787829954.429210:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:2.0:1787829954.429212:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:0.0:1787829954.429217:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000040:2.0:1787829954.429219:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000010:0.0:1787829954.429222:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82feff68.
00000100:00000001:2.0:1787829954.429223:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829954.429228:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000200:0.0:1787829954.429230:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327395712, offset 0
00000100:00000001:2.0:1787829954.429231:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829954.429236:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829954.429239:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429244:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429245:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.429249:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829954.429252:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787829954.429254:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:2.0:1787829954.429257:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429261:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1787829954.429264:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000008:00100000:2.0:1787829954.429265:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787829954.429269:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00000001:2.0:1787829954.429270:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829954.429274:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00000001:2.0:1787829954.429275:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829954.429278:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000010:2.0:1787829954.429282:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86115e00.
00000400:00000200:0.0:1787829954.429283:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000020:00000040:2.0:1787829954.429288:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000400:00000200:0.0:1787829954.429291:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40863:40863:256:4294967295] 192.168.203.116@tcp LPNI seq info [40863:40863:8:4294967295]
02000000:00000001:2.0:1787829954.429294:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829954.429297:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829954.429299:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829954.429301:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829954.429304:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829954.429306:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829954.429311:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000400:00000200:0.0:1787829954.429314:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787829954.429316:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829954.429323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1787829954.429328:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429334:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000010:0.0:1787829954.429336:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7c00.
00000100:00000001:2.0:1787829954.429336:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829954.429343:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001779a890] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1787829954.429347:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829954.429356:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:2.0:1787829954.429358:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829954.429361:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829954.429363:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829954.429365:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829954.429366:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829954.429367:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829954.429370:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4102
00000400:00000200:2.0:1787829954.429373:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829954.429374:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787829954.429375:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829954.429379:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787829954.429382:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829954.429385:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829954.429387:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298775212
00000100:00000001:2.0:1787829954.429388:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:0.0:1787829954.429392:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcca80.
00000100:00000040:2.0:1787829954.429393:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429398:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829954.429400:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000020:00000001:2.0:1787829954.429401:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829954.429404:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829954.429406:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829954.429408:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829954.429410:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000001:0.0:1787829954.429413:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298775212
00000020:00000001:2.0:1787829954.429413:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829954.429418:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429422:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:0.0:1787829954.429424:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc9c0.
00000100:00000040:2.0:1787829954.429425:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429430:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:0.0:1787829954.429431:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000040:2.0:1787829954.429433:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829954.429438:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829954.429440:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.429442:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787829954.429443:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829954.429446:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829954.429447:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829954.429449:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429452:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829954.429454:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829954.429455:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429456:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829954.429460:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829954.429461:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787829954.429462:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000100:00000001:0.0:1787829954.429467:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787829954.429467:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829954.429470:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429472:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1787829954.429477:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829954.429481:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
02000000:00000001:2.0:1787829954.429483:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1787829954.429484:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829954.429488:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787829954.429488:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429491:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000010:2.0:1787829954.429495:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000040:0.0:1787829954.429498:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829954.429502:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429512:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00100000:2.0:1787829954.429513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829954.429515:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429520:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829954.429523:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787829954.429527:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1787829954.429528:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429531:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829954.429537:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395840:192.168.203.116@tcp:38:kworker.0
00000100:00000040:2.0:1787829954.429538:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8bd533] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829954.429542:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829954.429544:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829954.429546:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429547:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829954.429552:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829954.429556:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829954.429559:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829954.429565:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4112)
00000400:00000200:0.0:1787829954.429571:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829954.429579:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877084d0.
00000400:00000010:0.0:1787829954.429583:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef3b8.
00000100:00000200:0.0:1787829954.429587:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395840, portal 10
00000100:00000001:0.0:1787829954.429589:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829954.429590:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402435200 : -116696307116416 : ffff95dd86115680)
00000100:00000040:0.0:1787829954.429597:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829954.429604:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829954.429606:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829954.429609:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef908.
00000100:00000200:0.0:1787829954.429612:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327395840, offset 0
00000400:00000200:0.0:1787829954.429614:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429617:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829954.429620:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829954.429622:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829954.429624:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829954.429625:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829954.429627:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429629:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40864:40864:255:4294967295] 192.168.203.116@tcp LPNI seq info [40864:40864:7:4294967295]
00000400:00000200:0.0:1787829954.429631:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829954.429635:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829954.429638:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829954.429640:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7600.
00000800:00000200:0.0:1787829954.429642:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829954.429645:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829954.429646:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829954.429649:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829954.429651:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829954.429652:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298774538
00000100:00000001:0.0:1787829954.429654:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298775212
00000100:00000001:0.0:1787829954.429678:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298774539
00000100:00000001:0.0:1787829954.429680:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298775212
00000100:00000001:0.0:1787829954.429681:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829954.429683:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429684:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829954.429687:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829954.429691:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829954.429692:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429693:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829954.429694:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829954.429696:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829954.429698:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829954.429699:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829954.429701:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829954.429702:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429703:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829954.429705:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829954.429707:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829954.429709:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829954.429711:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708d68.
00000400:00000010:0.0:1787829954.429714:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829954.429717:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327395968, portal 4
00000100:00000001:0.0:1787829954.429719:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829954.429720:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402437120 : -116696307114496 : ffff95dd86115e00)
00000100:00000040:0.0:1787829954.429725:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829954.429731:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829954.429733:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829954.429736:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fefaa0.
00000100:00000200:0.0:1787829954.429738:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327395968, offset 0
00000400:00000200:0.0:1787829954.429742:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429745:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829954.429749:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829954.429752:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829954.429780:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829954.429783:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829954.429785:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429787:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40865:40865:254:4294967295] 192.168.203.116@tcp LPNI seq info [40865:40865:6:4294967295]
00000400:00000200:0.0:1787829954.429791:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829954.429797:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829954.429801:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829954.429805:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7f00.
00000800:00000200:0.0:1787829954.429809:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829954.429813:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829954.429815:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829954.429817:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829954.429820:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829954.429822:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298774546
00000100:00000001:0.0:1787829954.429824:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298775212
00000100:00000001:0.0:1787829954.429826:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298774546
00000100:00000001:0.0:1787829954.429827:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298775212
00000100:00000001:0.0:1787829954.429829:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829954.429830:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429832:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429833:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829954.429835:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829954.429836:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829954.429837:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829954.429838:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829954.429839:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829954.429842:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829954.429846:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829954.429848:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429849:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829954.429850:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829954.429851:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829954.429853:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396096:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829954.429867:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829954.429869:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829954.429870:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829954.429872:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829954.429875:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829954.429877:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829954.429880:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829954.429882:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd877087e8.
00000400:00000010:0.0:1787829954.429885:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef7f8.
00000100:00000200:0.0:1787829954.429888:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396096, portal 4
00000100:00000001:0.0:1787829954.429890:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829954.429891:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402429440 : -116696307122176 : ffff95dd86114000)
00000100:00000040:0.0:1787829954.429898:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829954.429904:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829954.429906:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829954.429908:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fefb28.
00000100:00000200:0.0:1787829954.429911:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327396096, offset 0
00000400:00000200:0.0:1787829954.429915:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429918:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829954.429922:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829954.429924:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829954.429927:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829954.429929:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829954.429931:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829954.429934:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40866:40866:253:4294967295] 192.168.203.116@tcp LPNI seq info [40866:40866:5:4294967295]
00000400:00000200:0.0:1787829954.429937:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829954.429943:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829954.429947:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829954.429950:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7000.
00000800:00000200:0.0:1787829954.429954:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829954.429958:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000001:0.0:1787829954.429960:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829954.429962:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829954.429965:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829954.429967:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298774551
00000100:00000001:0.0:1787829954.429969:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298775212
00000100:00000001:0.0:1787829954.429971:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298774551
00000100:00000001:0.0:1787829954.429973:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298775212
00000100:00000001:0.0:1787829954.429975:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829954.429976:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429979:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429981:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829954.429983:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829954.429985:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787829954.429986:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829954.429988:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829954.429989:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.429990:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829954.940770:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829954.940777:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829954.940779:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829954.940784:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829954.940788:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829954.940791:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829954.940793:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829954.940795:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829954.940798:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829955.132787:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298775212, at 4298775232
00000100:00000010:3.0:1787829955.132802:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00000100:00000040:2.0:1787829955.132929:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298775212, at 4298775232
00000100:00000010:2.0:1787829955.132958:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00010000:00000001:0.0:1787829955.260744:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829955.260767:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829955.260770:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829955.260774:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260778:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829955.260781:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260783:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260786:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829955.260791:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829955.260792:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829955.260794:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260795:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829955.260796:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260798:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260800:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829955.260803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829955.260805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829955.260876:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829955.260880:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829955.260883:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829955.964766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829955.964771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829955.964773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829955.964777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829955.964779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829955.964782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829955.964784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829955.964785:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829955.964787:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829956.284790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829956.284797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829956.284803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829956.284805:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829956.284816:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829956.284817:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829956.284869:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829956.284873:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829956.284876:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829956.988791:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829956.988799:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829956.988801:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829956.988806:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829956.988809:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829956.988812:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829956.988815:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829956.988817:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829956.988820:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829957.308766:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829957.308772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829957.308777:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829957.308778:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829957.308781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829957.308782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829957.308801:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829957.308804:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829957.308807:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1787829957.500943:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.116:988
00000400:00000100:1.0:1787829957.500986:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.116@tcp at host 192.168.203.116 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1787829957.500995:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829957.500999:0:2336:0:(router.c:1791:lnet_notify()) 192.168.203.16@tcp notifying 192.168.203.116@tcp: down
00000400:00000200:1.0:1787829957.501003:0:2336:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829957.501007:0:2336:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1787829957.501012:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829957.501016:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7c00.
00000400:00000200:1.0:1787829957.501022:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829957.501026:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829957.501032:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82feff68
00000400:00000010:1.0:1787829957.501034:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82feff68.
00000100:00000001:1.0:1787829957.501039:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829957.501047:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501074:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829957.501078:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501085:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829957.501097:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829957.501106:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829957.501108:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7600.
00000400:00000200:1.0:1787829957.501111:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829957.501114:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829957.501116:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef908
00000400:00000010:1.0:1787829957.501118:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef908.
00000100:00000001:1.0:1787829957.501121:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829957.501124:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501131:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829957.501134:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501145:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829957.501147:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829957.501151:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829957.501153:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7f00.
00000400:00000200:1.0:1787829957.501155:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829957.501158:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829957.501161:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefaa0
00000400:00000010:1.0:1787829957.501163:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefaa0.
00000100:00000001:1.0:1787829957.501165:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829957.501170:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501176:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829957.501179:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501184:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829957.501186:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787829957.501189:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.116@tcp
00000800:00000010:1.0:1787829957.501191:0:2336:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7000.
00000400:00000200:1.0:1787829957.501193:0:2336:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787829957.501196:0:2336:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787829957.501198:0:2336:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fefb28
00000400:00000010:1.0:1787829957.501200:0:2336:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fefb28.
00000100:00000001:1.0:1787829957.501202:0:2336:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829957.501206:0:2336:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501213:0:2336:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829957.501216:0:2336:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829957.501222:0:2336:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829957.501223:0:2336:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829957.501298:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829957.501302:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829957.501304:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829957.501311:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829957]  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829957.501321:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829957.501323:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829957.501326:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877087e8.
00000400:00000200:0.0:1787829957.501329:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef7f8
00000400:00000010:0.0:1787829957.501331:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef7f8.
00000100:00000001:0.0:1787829957.501334:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829957.501337:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829957.501345:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501351:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829957.501352:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501355:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501357:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829957.501360:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501366:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829957.501371:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501388:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829957.501391:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501392:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501393:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501396:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829957.501398:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829957.501402:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829957.501406:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829957.501409:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829957.501413:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829957.501421:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396096:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829957.501425:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829957.501429:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829957]  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829957.501435:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829957.501437:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829957.501438:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd877084d0.
00000400:00000200:0.0:1787829957.501441:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef3b8
00000400:00000010:0.0:1787829957.501442:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef3b8.
00000100:00000001:0.0:1787829957.501444:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829957.501447:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829957.501454:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501460:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829957.501461:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501463:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501464:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829957.501468:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501474:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829957.501478:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501484:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829957.501486:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501487:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501488:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501490:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829957.501492:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829957.501494:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829957.501497:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829957.501499:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829957.501502:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829957.501509:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395840:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829957.501512:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829957.501515:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829957]  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829957.501522:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829957.501524:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829957.501525:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708d68.
00000400:00000200:0.0:1787829957.501527:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829957.501528:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829957.501530:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829957.501533:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829957.501540:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501546:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829957.501547:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501549:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501551:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829957.501554:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501560:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829957.501564:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501575:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829957.501577:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501579:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501580:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501582:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829957.501583:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829957.501585:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829957.501588:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829957.501590:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829957.501593:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829957.501599:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86115e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327395968:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829957.501603:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829957.501606:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829957]  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829957.501613:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829957.501614:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829957.501615:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829957.501618:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829957.501620:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829957.501622:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829957.501625:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829957.501631:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501636:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829957.501637:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501639:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501640:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829957.501643:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501650:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829957.501653:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501690:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829957.501692:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501694:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829957.501695:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501696:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829957.501709:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829957.501711:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829957.501715:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829957.501717:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829957.501721:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829957.501731:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327395712:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829957.501745:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501748:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829957.501751:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114000 x1874671327396096/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501758:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829957.501761:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829957.501764:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829957.501768:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829957.501771:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829957.501772:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829957.501775:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829957.501777:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:0.0:1787829957.501780:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829957.501782:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829957.501784:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114000.
00000100:00000001:0.0:1787829957.501787:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829957.501788:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501790:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829957.501793:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115680 x1874671327395840/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501799:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829957.501800:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829957.501802:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829957.501804:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829957.501806:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829957.501807:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829957.501808:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829957.501810:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:0.0:1787829957.501812:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829957.501813:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829957.501814:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115680.
00000100:00000001:0.0:1787829957.501816:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829957.501817:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501819:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829957.501822:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86115e00 x1874671327395968/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501828:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829957.501829:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829957.501830:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829957.501832:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829957.501833:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829957.501834:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829957.501836:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829957.501837:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:0.0:1787829957.501839:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829957.501840:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829957.501841:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86115e00.
00000100:00000001:0.0:1787829957.501843:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829957.501844:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501845:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829957.501848:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116580 x1874671327395712/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829957.501853:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829957.501854:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829957.501855:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530800.
02000000:00000001:0.0:1787829957.501858:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829957.501858:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829957.501859:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829957.501861:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829957.501862:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003400.
02000000:00000001:0.0:1787829957.501864:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829957.501866:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829957.501867:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116580.
00000100:00000001:0.0:1787829957.501869:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829957.501870:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501873:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829957.501879:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829957.501881:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829957.501884:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829957.501885:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829958.012768:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829958.012774:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829958.012775:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829958.012779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829958.012783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829958.012786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829958.012788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829958.012791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829958.012794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829958.332780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829958.332787:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829958.332791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829958.332793:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829958.332797:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829958.332798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829958.332881:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829958.332884:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829958.332887:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829959.036786:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829959.036792:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829959.036794:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829959.036799:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829959.036802:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829959.036806:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.036808:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829959.036810:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.036813:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829959.356784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829959.356797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829959.356802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829959.356804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829959.356808:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829959.356810:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829959.356864:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829959.356867:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829959.356869:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787829959.548835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829959.548845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787829959.548848:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829959.548852:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829959.548857:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829959.548864:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4107
00000400:00000200:1.0:1787829959.548874:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829959.548881:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829959.548885:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829959.548889:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.548893:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.548895:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.548898:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829959.548902:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829959.548903:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:1.0:1787829959.548906:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829959.548909:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:1.0:1787829959.548912:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:1.0:1787829959.548914:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:1.0:1787829959.548917:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.548919:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.548921:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.548923:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829959.548926:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:1.0:1787829959.548929:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829959.548931:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829959.548934:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829959.548937:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829959.548939:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.548942:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829959.548944:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829959.548950:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e21c0.
00000020:00000040:1.0:1787829959.548953:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:1.0:1787829959.548957:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829959.548958:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.548961:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829959.548969:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829959.548973:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829959.548979:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f4000.
00000100:00000001:1.0:1787829959.548985:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829959.548992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829959.549001:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829959.549006:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829959.549009:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e571cad] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829959.549040:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829959.549044:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829959.549047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787829959.549048:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829959.549050:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829959.549053:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829959.549055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4107
00000400:00000200:1.0:1787829959.549059:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829959.549062:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829959.549064:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829959.549066:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549069:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.549070:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549072:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829959.549074:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829959.549075:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:1.0:1787829959.549077:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829959.549080:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:1.0:1787829959.549082:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:1.0:1787829959.549084:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:1.0:1787829959.549086:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549088:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.549090:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549092:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829959.549094:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:1.0:1787829959.549097:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549099:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829959.549101:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829959.549106:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829959.549108:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549110:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829959.549113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829959.549117:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549119:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829959.549123:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e0000.
00000020:00000040:1.0:1787829959.549125:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:1.0:1787829959.549127:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829959.549128:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549130:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829959.549131:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829959.549133:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829959.549136:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f7000.
00000100:00000001:1.0:1787829959.549140:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829959.549145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829959.549152:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829959.549154:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829959.549158:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbfbd16] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829959.549166:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829959.549168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829959.549172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1787829959.549173:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829959.549174:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829959.549176:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829959.549179:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4107
00000400:00000200:1.0:1787829959.549182:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829959.549185:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829959.549187:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829959.549190:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549192:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.549194:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549196:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829959.549198:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829959.549199:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:1.0:1787829959.549200:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829959.549202:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:1.0:1787829959.549204:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:1.0:1787829959.549206:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:1.0:1787829959.549209:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549211:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.549212:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549214:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829959.549217:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:1.0:1787829959.549219:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549222:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829959.549224:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829959.549227:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829959.549229:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549230:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829959.549233:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829959.549235:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549237:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829959.549240:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e12c0.
00000020:00000040:1.0:1787829959.549242:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:1.0:1787829959.549244:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829959.549245:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549247:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829959.549248:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829959.549250:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829959.549253:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f6000.
00000100:00000001:1.0:1787829959.549256:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829959.549261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829959.549268:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829959.549270:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829959.549273:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d6eb3768] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829959.549280:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787829959.549282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787829959.549286:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787829959.549287:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787829959.549288:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787829959.549291:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787829959.549293:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4107
00000400:00000200:1.0:1787829959.549296:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829959.549299:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:1.0:1787829959.549301:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787829959.549304:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549311:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.549313:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549315:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:1.0:1787829959.549327:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787829959.549328:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:1.0:1787829959.549330:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787829959.549332:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:1.0:1787829959.549334:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:1.0:1787829959.549336:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:1.0:1787829959.549338:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549340:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829959.549342:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549344:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:1.0:1787829959.549346:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:1.0:1787829959.549348:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829959.549351:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829959.549353:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829959.549355:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829959.549357:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549359:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787829959.549361:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787829959.549363:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549365:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787829959.549368:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd870e3c00.
00000020:00000040:1.0:1787829959.549370:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:1.0:1787829959.549372:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829959.549373:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829959.549375:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829959.549376:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829959.549378:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829959.549382:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd860f5800.
00000100:00000001:1.0:1787829959.549386:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829959.549390:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829959.549396:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787829959.549398:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829959.549401:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ffb96165] to pc [ptlrpcd_rcv+-1]  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829959.549408:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829959.549413:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4117)
00000100:00000001:0.0:1787829959.549587:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829959.549591:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829959.549594:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829959.549601:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549610:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829959.549612:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.549615:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829959.549617:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829959.549620:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829959.549623:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327396224:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829959.549627:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829959.549631:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829959.549633:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.549636:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829959.549641:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829959.549643:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829959.549649:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829959.549653:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829959.549680:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829959.549688:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396224, portal 25
00000100:00000001:0.0:1787829959.549690:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829959.549692:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419002304 : -116696290549312 : ffff95dd870e21c0)
00000100:00000040:0.0:1787829959.549698:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549705:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829959.549708:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829959.549711:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829959.549714:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327396224, offset 0
00000400:00000200:0.0:1787829959.549731:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829959.549739:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.549744:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829959.549749:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829959.549754:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829959.549757:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829959.549760:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829959.549764:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40867:40867:256:4294967295] 192.168.203.116@tcp LPNI seq info [40867:40867:8:4294967295]
00000400:00000200:0.0:1787829959.549767:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.549778:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829959.549783:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.549789:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829959.549795:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829959.549799:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829959.549803:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829959.549805:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829959.549809:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.549811:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829959.549815:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829959.549819:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829959.549823:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829959.549825:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829959.549828:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829959.549831:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549838:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.549842:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549848:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.549849:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829959.549855:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829959.549858:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829959.549861:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298780333
00000100:00000010:0.0:1787829959.549865:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc600.
00000100:00000001:0.0:1787829959.549872:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829959.549876:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298780333
00000100:00000010:0.0:1787829959.549879:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcca80.
00000100:00000001:0.0:1787829959.549882:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829959.549890:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829959.549893:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.549895:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829959.549899:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959]  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829959.549908:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829959.549909:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829959.549911:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829959.549914:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829959.549915:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829959.549917:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829959.549920:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829959.549927:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549932:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829959.549933:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.549935:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.549937:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829959.549940:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549946:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829959.549950:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.549956:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829959.549958:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.549972:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.549974:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.549976:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829959.549977:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829959.549980:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829959.549983:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829959.549986:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829959.549990:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829959.549996:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327396224:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829959.550000:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829959.550003:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550009:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829959.550010:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.550012:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829959.550013:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829959.550015:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829959.550018:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396352:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829959.550020:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829959.550022:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829959.550023:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.550025:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550028:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829959.550030:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829959.550033:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829959.550035:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829959.550038:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829959.550041:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396352, portal 10
00000100:00000001:0.0:1787829959.550043:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829959.550044:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418993664 : -116696290557952 : ffff95dd870e0000)
00000100:00000040:0.0:1787829959.550050:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550055:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829959.550057:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829959.550059:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829959.550061:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327396352, offset 0
00000400:00000200:0.0:1787829959.550065:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829959.550069:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.550072:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829959.550075:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829959.550078:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829959.550080:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829959.550081:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829959.550084:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40868:40868:256:4294967295] 192.168.203.116@tcp LPNI seq info [40868:40868:8:4294967295]
00000400:00000200:0.0:1787829959.550087:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.550093:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829959.550097:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.550100:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829959.550103:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829959.550106:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829959.550109:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829959.550111:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829959.550115:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.550117:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829959.550119:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829959.550122:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829959.550125:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829959.550126:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829959.550128:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829959.550131:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550138:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550141:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550146:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.550148:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829959.550150:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829959.550155:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829959.550157:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829959.550159:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298779659
00000100:00000001:0.0:1787829959.550161:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298780333
00000100:00000001:0.0:1787829959.550163:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298779659
00000100:00000001:0.0:1787829959.550165:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298780333
00000100:00000001:0.0:1787829959.550167:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829959.550169:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.550171:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829959.550175:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959]  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829959.550181:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829959.550182:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829959.550183:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829959.550185:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829959.550187:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829959.550189:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829959.550191:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829959.550199:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550204:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829959.550205:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550207:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.550208:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829959.550211:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550218:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829959.550222:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550228:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829959.550230:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550232:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.550233:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550235:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829959.550237:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829959.550239:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829959.550242:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829959.550245:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829959.550248:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829959.550256:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396352:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829959.550259:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829959.550262:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550268:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829959.550269:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.550271:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829959.550272:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829959.550274:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829959.550277:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396480:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829959.550281:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829959.550283:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829959.550284:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.550286:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550289:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829959.550291:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829959.550295:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829959.550297:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829959.550300:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829959.550303:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396480, portal 4
00000100:00000001:0.0:1787829959.550305:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829959.550306:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377418998464 : -116696290553152 : ffff95dd870e12c0)
00000100:00000040:0.0:1787829959.550311:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550317:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829959.550319:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829959.550322:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829959.550324:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327396480, offset 0
00000400:00000200:0.0:1787829959.550328:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829959.550332:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.550336:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829959.550338:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829959.550341:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829959.550343:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829959.550345:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829959.550348:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40869:40869:256:4294967295] 192.168.203.116@tcp LPNI seq info [40869:40869:8:4294967295]
00000400:00000200:0.0:1787829959.550352:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.550358:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829959.550362:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.550365:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829959.550369:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829959.550371:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829959.550375:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829959.550377:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829959.550394:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.550396:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829959.550399:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829959.550402:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829959.550405:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829959.550407:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829959.550409:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829959.550413:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550420:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550423:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550429:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.550431:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829959.550434:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829959.550436:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829959.550440:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829959.550442:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298779666
00000100:00000001:0.0:1787829959.550444:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298780333
00000100:00000001:0.0:1787829959.550446:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298779666
00000100:00000001:0.0:1787829959.550448:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298780333
00000100:00000001:0.0:1787829959.550450:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829959.550452:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.550453:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829959.550457:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959]  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829959.550463:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829959.550465:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829959.550467:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829959.550469:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829959.550471:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829959.550473:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829959.550476:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829959.550485:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550497:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829959.550498:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550501:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.550502:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829959.550505:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550511:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829959.550514:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550521:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829959.550523:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550524:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.550525:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550527:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829959.550528:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829959.550530:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829959.550534:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829959.550536:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829959.550540:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829959.550548:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396480:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829959.550552:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829959.550555:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550561:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829959.550563:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.550565:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829959.550567:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829959.550569:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829959.550572:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396608:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829959.550575:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829959.550576:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829959.550578:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829959.550580:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550583:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829959.550585:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829959.550588:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829959.550590:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829959.550593:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829959.550595:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396608, portal 4
00000100:00000001:0.0:1787829959.550597:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829959.550599:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419009024 : -116696290542592 : ffff95dd870e3c00)
00000100:00000040:0.0:1787829959.550604:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550611:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829959.550613:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829959.550615:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829959.550618:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327396608, offset 0
00000400:00000200:0.0:1787829959.550622:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829959.550626:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.550629:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829959.550632:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829959.550635:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829959.550637:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829959.550639:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829959.550642:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40870:40870:256:4294967295] 192.168.203.116@tcp LPNI seq info [40870:40870:8:4294967295]
00000400:00000200:0.0:1787829959.550646:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829959.550652:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829959.550685:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.550688:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829959.550692:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829959.550694:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787829959.550698:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829959.550700:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787829959.550704:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829959.550705:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829959.550708:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829959.550712:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829959.550714:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829959.550716:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829959.550718:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829959.550735:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550743:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550746:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550752:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.550754:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829959.550757:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829959.550760:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829959.550763:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829959.550765:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298779672
00000100:00000001:0.0:1787829959.550767:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298780333
00000100:00000001:0.0:1787829959.550769:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298779672
00000100:00000001:0.0:1787829959.550771:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298780333
00000100:00000001:0.0:1787829959.550772:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829959.550774:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.550776:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829959.550779:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959]  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829959.550787:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829959.550788:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829959.550790:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829959.550792:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829959.550794:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829959.550796:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829959.550799:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829959.550806:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550812:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829959.550814:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550815:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.550817:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829959.550819:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550825:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829959.550829:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550836:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829959.550838:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550839:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829959.550841:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550843:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829959.550845:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829959.550847:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829959.550851:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829959.550853:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829959.550858:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829959.550866:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd870e3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396608:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829959.550871:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550874:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550878:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e21c0 x1874671327396224/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550884:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829959.550888:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550890:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829959.550894:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829959.550897:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829959.550899:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829959.550902:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829959.550904:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f4000.
02000000:00000001:0.0:1787829959.550907:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829959.550909:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829959.550911:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e21c0.
00000100:00000001:0.0:1787829959.550913:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829959.550915:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550916:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550919:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e0000 x1874671327396352/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550925:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829959.550926:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550927:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829959.550930:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829959.550931:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829959.550932:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829959.550933:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829959.550935:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f7000.
02000000:00000001:0.0:1787829959.550937:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829959.550938:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829959.550940:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e0000.
00000100:00000001:0.0:1787829959.550942:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829959.550943:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550945:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550953:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e12c0 x1874671327396480/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550959:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829959.550960:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550961:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531c00.
02000000:00000001:0.0:1787829959.550963:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829959.550964:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829959.550965:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829959.550967:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829959.550969:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f6000.
02000000:00000001:0.0:1787829959.550970:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829959.550972:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829959.550973:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e12c0.
00000100:00000001:0.0:1787829959.550975:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829959.550976:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.550978:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829959.550981:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd870e3c00 x1874671327396608/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829959.550987:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829959.550989:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829959.550990:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531000.
02000000:00000001:0.0:1787829959.550992:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829959.550993:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829959.550994:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829959.550996:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829959.550997:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd860f5800.
02000000:00000001:0.0:1787829959.550999:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829959.551000:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829959.551002:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd870e3c00.
00000100:00000001:0.0:1787829959.551004:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829959.551005:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.551007:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829959.551015:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829959.551017:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829959.551019:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829959.551021:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829960.060764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829960.060771:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829960.060773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829960.060778:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829960.060782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829960.060785:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829960.060788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829960.060790:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829960.060793:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829960.252776:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298780333, at 4298780352
00000100:00000040:3.0:1787829960.252780:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298780333, at 4298780352
00000100:00000010:2.0:1787829960.252792:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc600.
00000100:00000010:3.0:1787829960.252797:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcca80.
00010000:00000001:0.0:1787829960.380767:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829960.380774:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829960.380780:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829960.380781:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829960.380785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829960.380788:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829960.380790:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829960.380794:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829960.380797:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829960.380800:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829960.380802:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829960.380804:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829960.380822:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829960.380825:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829960.380828:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829961.084756:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829961.084760:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829961.084762:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829961.084765:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829961.084768:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829961.084771:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829961.084773:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829961.084775:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829961.084776:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829961.404762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829961.404766:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829961.404771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829961.404772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829961.404774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829961.404775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829961.404821:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829961.404826:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829961.404829:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829962.108802:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829962.108809:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829962.108812:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829962.108817:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829962.108821:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829962.108824:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829962.108828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829962.108831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829962.108834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829962.428832:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829962.428838:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829962.428844:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829962.428845:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829962.428848:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829962.428849:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787829962.428904:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1787829962.428909:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1787829962.428912:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829963.132976:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829963.132983:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829963.132985:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829963.132990:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829963.132994:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829963.132997:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829963.132999:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829963.133002:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829963.133005:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829963.452785:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829963.452797:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829963.452803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829963.452804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829963.452807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829963.452809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829963.452824:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829963.452826:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829963.452828:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829964.156773:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829964.156779:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829964.156782:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829964.156787:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829964.156790:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829964.156794:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.156796:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829964.156799:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.156802:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829964.476792:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829964.476799:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829964.476803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829964.476804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829964.476807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829964.476809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829964.476854:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829964.476858:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829964.476862:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829964.669896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829964.669905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829964.669907:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829964.669911:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829964.669914:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829964.669921:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4112
00000400:00000200:3.0:1787829964.669928:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829964.669934:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829964.669937:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829964.669940:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.669943:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.669945:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.669948:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829964.669952:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829964.669954:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829964.669956:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829964.669959:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829964.669962:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829964.669964:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829964.669967:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.669969:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.669970:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.669972:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829964.669975:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829964.669978:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829964.669980:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829964.669982:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829964.669985:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829964.669988:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.669991:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829964.669993:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829964.670001:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed30c0.
00000020:00000040:3.0:1787829964.670004:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829964.670007:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829964.670009:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670012:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829964.670015:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829964.670018:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829964.670025:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3800.
00000100:00000001:3.0:1787829964.670030:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829964.670038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829964.670047:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829964.670051:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829964.670055:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000988b3b52] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829964.670080:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829964.670083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829964.670086:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829964.670087:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829964.670089:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829964.670091:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829964.670093:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4112
00000400:00000200:3.0:1787829964.670096:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829964.670099:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829964.670101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829964.670104:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670106:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.670108:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670110:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829964.670112:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829964.670113:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829964.670115:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829964.670117:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829964.670119:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829964.670121:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829964.670123:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670125:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.670127:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670128:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829964.670130:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:3.0:1787829964.670132:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670134:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829964.670136:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829964.670140:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829964.670142:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670144:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829964.670147:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829964.670150:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670152:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829964.670155:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0b40.
00000020:00000040:3.0:1787829964.670156:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829964.670158:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829964.670159:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670161:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829964.670162:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829964.670164:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829964.670168:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0c00.
00000100:00000001:3.0:1787829964.670170:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829964.670175:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829964.670182:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829964.670184:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829964.670188:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cfb7ee7b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829964.670195:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829964.670197:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829964.670199:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829964.670200:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829964.670201:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829964.670203:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829964.670205:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4112
00000400:00000200:3.0:1787829964.670208:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829964.670210:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829964.670212:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829964.670214:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670216:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.670218:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670220:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829964.670221:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829964.670222:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829964.670223:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829964.670225:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829964.670227:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829964.670228:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:3.0:1787829964.670231:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670232:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.670234:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670236:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829964.670238:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:3.0:1787829964.670240:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670242:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829964.670244:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829964.670246:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829964.670248:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670249:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829964.670251:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829964.670253:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670254:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829964.670256:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829964.670258:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829964.670259:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829964.670261:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670262:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829964.670263:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829964.670264:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829964.670267:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a1800.
00000100:00000001:3.0:1787829964.670269:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829964.670273:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829964.670279:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829964.670281:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829964.670284:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829964.670290:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829964.670292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829964.670295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829964.670296:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829964.670297:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829964.670298:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829964.670300:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4112
00000400:00000200:3.0:1787829964.670303:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829964.670305:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829964.670307:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829964.670309:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670310:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.670312:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670313:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829964.670315:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829964.670316:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:3.0:1787829964.670317:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829964.670318:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:3.0:1787829964.670320:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829964.670322:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:3.0:1787829964.670324:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670325:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829964.670327:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670328:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829964.670330:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829964.670332:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829964.670334:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829964.670335:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829964.670337:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829964.670338:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670339:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829964.670341:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829964.670343:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670344:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829964.670346:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829964.670348:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:3.0:1787829964.670350:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829964.670350:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829964.670352:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829964.670353:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829964.670354:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829964.670357:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a0400.
00000100:00000001:3.0:1787829964.670360:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829964.670363:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829964.670368:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829964.670370:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829964.670372:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829964.670378:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829964.670381:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4122)
00000100:00000001:0.0:1787829964.670450:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829964.670455:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829964.670457:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829964.670465:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670473:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829964.670476:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.670479:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829964.670481:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829964.670484:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829964.670487:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327396736:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829964.670491:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829964.670495:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829964.670498:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.670500:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829964.670504:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829964.670507:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829964.670511:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829964.670515:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829964.670520:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829964.670526:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396736, portal 25
00000100:00000001:0.0:1787829964.670529:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829964.670531:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400066240 : -116696309485376 : ffff95dd85ed30c0)
00000100:00000040:0.0:1787829964.670536:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670543:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829964.670546:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829964.670548:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829964.670552:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327396736, offset 0
00000400:00000200:0.0:1787829964.670557:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829964.670564:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.670569:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829964.670574:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829964.670578:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829964.670581:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829964.670584:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829964.670588:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40871:40871:256:4294967295] 192.168.203.116@tcp LPNI seq info [40871:40871:8:4294967295]
00000400:00000200:0.0:1787829964.670592:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.670601:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829964.670607:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.670612:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829964.670618:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829964.670622:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829964.670627:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829964.670631:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829964.670635:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.670638:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829964.670643:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829964.670647:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829964.670652:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829964.670676:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829964.670681:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829964.670695:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670710:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.670714:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670720:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.670722:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829964.670728:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829964.670731:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829964.670733:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298785454
00000100:00000010:0.0:1787829964.670738:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc9c0.
00000100:00000001:0.0:1787829964.670745:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787829964.670749:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298785454
00000100:00000010:0.0:1787829964.670751:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd87fcc0c0.
00000100:00000001:0.0:1787829964.670754:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787829964.670766:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829964.670769:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.670772:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829964.670776:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964]  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829964.670785:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829964.670787:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829964.670789:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829964.670791:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829964.670793:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829964.670795:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829964.670799:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829964.670807:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670813:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829964.670815:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.670817:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.670819:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829964.670823:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670829:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829964.670833:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670840:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829964.670842:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.670844:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.670845:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.670847:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829964.670849:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829964.670852:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829964.670855:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829964.670857:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829964.670861:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829964.670868:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327396736:192.168.203.116@tcp:250:kworker.0
00000100:00000001:0.0:1787829964.670872:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829964.670875:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670881:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829964.670882:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.670884:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829964.670885:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829964.670887:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829964.670889:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396864:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829964.670892:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829964.670894:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829964.670895:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.670897:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829964.670900:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829964.670901:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829964.670904:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829964.670907:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829964.670910:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829964.670912:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396864, portal 10
00000100:00000001:0.0:1787829964.670914:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829964.670915:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400056640 : -116696309494976 : ffff95dd85ed0b40)
00000100:00000040:0.0:1787829964.670921:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.670926:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829964.670928:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829964.670930:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829964.670932:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327396864, offset 0
00000400:00000200:0.0:1787829964.670935:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829964.670939:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.670942:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829964.670945:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829964.670947:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829964.670949:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829964.670951:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829964.670954:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40872:40872:256:4294967295] 192.168.203.116@tcp LPNI seq info [40872:40872:8:4294967295]
00000400:00000200:0.0:1787829964.670957:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.670962:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829964.670966:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.670969:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829964.670972:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829964.670975:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829964.670978:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829964.670981:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829964.670984:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.670986:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829964.670988:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829964.670991:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829964.670994:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829964.670995:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829964.670997:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829964.671001:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671007:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671010:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671015:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671016:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829964.671018:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829964.671022:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:0.0:1787829964.671024:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829964.671026:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298784780
00000100:00000001:0.0:1787829964.671028:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298785454
00000100:00000001:0.0:1787829964.671030:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298784780
00000100:00000001:0.0:1787829964.671032:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298785454
00000100:00000001:0.0:1787829964.671034:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829964.671035:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671037:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829964.671041:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964]  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829964.671048:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829964.671049:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829964.671051:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829964.671054:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829964.671055:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829964.671057:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829964.671060:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829964.671068:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671074:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829964.671075:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671076:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.671077:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829964.671081:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671088:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829964.671091:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671098:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829964.671100:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671102:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.671103:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671105:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829964.671106:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829964.671108:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829964.671111:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829964.671113:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829964.671117:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829964.671125:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396864:192.168.203.116@tcp:38:kworker.0
00000100:00000001:0.0:1787829964.671128:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829964.671131:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671137:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829964.671139:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.671141:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829964.671142:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829964.671144:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829964.671147:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829964.671150:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829964.671151:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829964.671153:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.671155:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829964.671157:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829964.671160:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829964.671162:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829964.671164:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829964.671167:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829964.671170:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327396992, portal 4
00000100:00000001:0.0:1787829964.671171:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829964.671173:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:0.0:1787829964.671178:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671184:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829964.671186:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829964.671188:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829964.671190:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327396992, offset 0
00000400:00000200:0.0:1787829964.671193:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829964.671196:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.671199:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829964.671202:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829964.671204:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829964.671206:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829964.671208:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829964.671211:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40873:40873:256:4294967295] 192.168.203.116@tcp LPNI seq info [40873:40873:8:4294967295]
00000400:00000200:0.0:1787829964.671215:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.671221:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829964.671225:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.671228:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829964.671231:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829964.671234:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829964.671238:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829964.671240:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829964.671244:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.671246:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829964.671248:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829964.671252:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829964.671254:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829964.671256:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829964.671258:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829964.671261:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671268:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671271:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671277:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671279:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829964.671281:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829964.671283:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:0.0:1787829964.671286:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829964.671288:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298784787
00000100:00000001:0.0:1787829964.671290:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298785454
00000100:00000001:0.0:1787829964.671292:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298784787
00000100:00000001:0.0:1787829964.671294:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298785454
00000100:00000001:0.0:1787829964.671296:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829964.671298:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671299:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829964.671303:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964]  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829964.671310:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829964.671311:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829964.671313:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829964.671315:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829964.671317:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829964.671318:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829964.671321:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829964.671329:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671335:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829964.671337:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671338:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.671339:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829964.671342:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671350:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829964.671354:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671361:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829964.671363:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671364:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.671366:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671368:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829964.671369:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829964.671372:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829964.671375:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829964.671377:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829964.671380:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829964.671386:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327396992:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829964.671390:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829964.671393:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671399:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829964.671401:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.671402:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787829964.671404:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829964.671406:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829964.671409:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397120:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829964.671412:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787829964.671413:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787829964.671414:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829964.671416:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787829964.671418:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829964.671420:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787829964.671423:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:0.0:1787829964.671425:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd87708528.
00000400:00000010:0.0:1787829964.671428:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef6e8.
00000100:00000200:0.0:1787829964.671430:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397120, portal 4
00000100:00000001:0.0:1787829964.671432:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787829964.671433:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:0.0:1787829964.671438:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671445:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787829964.671447:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:0.0:1787829964.671449:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd82fef198.
00000100:00000200:0.0:1787829964.671451:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327397120, offset 0
00000400:00000200:0.0:1787829964.671454:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:0.0:1787829964.671457:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.671460:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:0.0:1787829964.671462:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787829964.671464:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:0.0:1787829964.671466:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787829964.671468:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:0.0:1787829964.671471:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40874:40874:256:4294967295] 192.168.203.116@tcp LPNI seq info [40874:40874:8:4294967295]
00000400:00000200:0.0:1787829964.671474:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829964.671480:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787829964.671484:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.671487:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd941d7800.
00000800:00000200:0.0:1787829964.671490:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.0:1787829964.671493:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787829964.671496:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:0.1:1787829964.671499:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1787829964.671503:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:0.0:1787829964.671505:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd941d7800.
00000400:00000200:0.0:1787829964.671508:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787829964.671511:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787829964.671513:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef198
00000400:00000010:0.0:1787829964.671514:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef198.
00000100:00000001:0.0:1787829964.671517:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787829964.671520:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671527:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671530:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671541:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671543:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829964.671545:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829964.671547:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:0.0:1787829964.671550:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:0.0:1787829964.671552:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298784792
00000100:00000001:0.0:1787829964.671554:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298785454
00000100:00000001:0.0:1787829964.671556:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298784792
00000100:00000001:0.0:1787829964.671558:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298785454
00000100:00000001:0.0:1787829964.671560:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787829964.671562:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671563:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787829964.671566:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964]  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787829964.671572:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787829964.671573:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787829964.671575:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd87708528.
00000400:00000200:0.0:1787829964.671577:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd82fef6e8
00000400:00000010:0.0:1787829964.671578:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd82fef6e8.
00000100:00000001:0.0:1787829964.671580:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787829964.671583:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787829964.671590:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671596:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787829964.671598:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671599:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.671601:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787829964.671603:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671610:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829964.671614:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671620:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787829964.671622:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671624:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787829964.671625:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671627:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787829964.671628:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787829964.671631:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787829964.671633:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:0.0:1787829964.671635:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787829964.671639:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787829964.671645:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397120:192.168.203.116@tcp:8:kworker.0
00000100:00000001:0.0:1787829964.671649:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671651:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671654:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed30c0 x1874671327396736/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671676:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829964.671679:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829964.671681:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90531400.
02000000:00000001:0.0:1787829964.671684:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829964.671698:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829964.671699:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:0.0:1787829964.671702:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829964.671704:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3800.
02000000:00000001:0.0:1787829964.671708:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829964.671710:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829964.671712:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed30c0.
00000100:00000001:0.0:1787829964.671714:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829964.671715:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671717:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671720:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0b40 x1874671327396864/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671727:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829964.671728:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829964.671730:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90532000.
02000000:00000001:0.0:1787829964.671731:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829964.671732:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829964.671734:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:0.0:1787829964.671736:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829964.671737:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0c00.
02000000:00000001:0.0:1787829964.671739:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829964.671740:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829964.671742:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0b40.
00000100:00000001:0.0:1787829964.671743:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829964.671744:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671746:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671748:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327396992/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671754:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829964.671755:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829964.671756:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90530400.
02000000:00000001:0.0:1787829964.671758:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829964.671759:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829964.671760:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:0.0:1787829964.671762:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829964.671763:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a1800.
02000000:00000001:0.0:1787829964.671765:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829964.671766:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829964.671767:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:0.0:1787829964.671770:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829964.671771:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671772:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787829964.671775:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327397120/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829964.671782:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787829964.671783:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787829964.671785:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd90533000.
02000000:00000001:0.0:1787829964.671787:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787829964.671788:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787829964.671789:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:0.0:1787829964.671790:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787829964.671791:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a0400.
02000000:00000001:0.0:1787829964.671793:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787829964.671794:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787829964.671795:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:0.0:1787829964.671797:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787829964.671798:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671800:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829964.671806:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829964.671808:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829964.671810:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787829964.671811:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829965.180891:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829965.180898:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829965.180900:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829965.180905:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829965.180908:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829965.180911:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829965.180914:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829965.180916:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829965.180919:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829965.372772:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298785454, at 4298785472
00000100:00000040:3.0:1787829965.372772:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298785454, at 4298785472
00000100:00000010:3.0:1787829965.372786:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc0c0.
00000100:00000010:2.0:1787829965.372788:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd87fcc9c0.
00010000:00000001:0.0:1787829965.500765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829965.500772:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829965.500774:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829965.500777:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500781:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829965.500783:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500785:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500787:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500790:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829965.500791:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829965.500793:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829965.500794:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500795:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829965.500797:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500798:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500800:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829965.500803:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829965.500804:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829965.500880:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829965.500883:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829965.500886:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829966.204775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829966.204781:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829966.204783:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829966.204788:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829966.204792:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829966.204795:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829966.204798:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829966.204800:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829966.204803:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829966.524770:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829966.524776:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829966.524781:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829966.524783:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829966.524786:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829966.524788:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829966.524806:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829966.524809:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829966.524812:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829967.230806:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829967.230812:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829967.230814:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829967.230819:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829967.230822:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829967.230825:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829967.230828:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829967.230831:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829967.230834:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829967.548782:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829967.548789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829967.548794:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829967.548796:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829967.548799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829967.548801:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829967.549226:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829967.549228:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829967.549231:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829968.253232:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829968.253239:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829968.253241:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829968.253246:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829968.253250:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829968.253253:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829968.253256:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829968.253258:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829968.253261:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829968.572814:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829968.572821:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829968.572827:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829968.572829:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829968.572834:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829968.572837:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829968.572987:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829968.572994:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829968.572999:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829969.276866:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829969.276941:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829969.276945:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829969.276951:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829969.276956:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829969.276968:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829969.276971:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829969.276974:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829969.276977:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829969.596840:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829969.596850:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829969.596858:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829969.596859:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829969.596863:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829969.596864:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829969.596896:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829969.596899:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829969.596904:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787829969.789843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829969.789853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787829969.789856:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829969.789859:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829969.789863:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829969.789870:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4117
00000400:00000200:0.0:1787829969.789879:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829969.789884:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829969.789887:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829969.789890:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.789893:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829969.789895:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.789897:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829969.789901:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829969.789902:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:0.0:1787829969.789905:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829969.789908:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:0.0:1787829969.789910:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:0.0:1787829969.789912:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:0.0:1787829969.789915:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.789917:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829969.789919:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.789920:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829969.789923:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:0.0:1787829969.789926:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829969.789928:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829969.789931:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829969.789934:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829969.789937:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.789939:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787829969.789941:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829969.789949:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716de00.
00000020:00000040:0.0:1787829969.789953:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:0.0:1787829969.789956:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829969.789958:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.789960:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787829969.789963:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829969.789966:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787829969.789972:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90532000.
00000100:00000001:0.0:1787829969.789978:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829969.789987:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787829969.789997:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829969.790002:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829969.790006:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c17e5d1d] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829969.790030:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829969.790033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829969.790036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787829969.790037:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787829969.790038:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829969.790041:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829969.790044:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4117
00000400:00000200:0.0:1787829969.790048:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829969.790051:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829969.790053:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829969.790056:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790059:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829969.790062:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790064:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:0.0:1787829969.790067:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829969.790068:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:0.0:1787829969.790070:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787829969.790071:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787829969.790072:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:0.0:1787829969.790076:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829969.790077:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787829969.790078:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829969.790080:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787829969.790084:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790088:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787829969.790088:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829969.790093:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790097:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829969.790102:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:1.0:1787829969.790103:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829969.790106:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829969.790108:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
02000000:00000001:1.0:1787829969.790110:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787829969.790111:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.790113:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829969.790115:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829969.790116:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829969.790120:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327397248:192.168.203.116@tcp:250:kworker.0
02000000:00000001:0.0:1787829969.790121:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829969.790126:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790128:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787829969.790129:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787829969.790133:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1787829969.790134:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829969.790137:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787829969.790139:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.790141:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829969.790147:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829969.790150:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790155:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.790156:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000010:0.0:1787829969.790162:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716ed00.
00000400:00000010:1.0:1787829969.790163:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000020:00000040:0.0:1787829969.790167:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000400:00000010:1.0:1787829969.790172:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
02000000:00000001:0.0:1787829969.790175:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829969.790178:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790180:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1787829969.790180:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397248, portal 25
02000000:00000001:0.0:1787829969.790183:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787829969.790185:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787829969.790186:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790187:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419574784 : -116696289976832 : ffff95dd8716de00)
02000000:00000010:0.0:1787829969.790193:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90531400.
00000100:00000040:1.0:1787829969.790194:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829969.790199:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790201:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829969.790205:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00100000:0.0:1787829969.790208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787829969.790209:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829969.790215:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327397248, offset 0
00000100:00000001:0.0:1787829969.790222:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787829969.790223:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790226:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787829969.790232:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008f0359f0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829969.790233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.790241:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:0.0:1787829969.790244:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.790246:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1787829969.790249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829969.790252:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000100:00080000:0.0:1787829969.790256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1787829969.790257:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787829969.790259:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787829969.790261:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00080000:0.0:1787829969.790262:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829969.790267:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40875:40875:256:4294967295] 192.168.203.116@tcp LPNI seq info [40875:40875:8:4294967295]
00000100:00000001:0.0:1787829969.790268:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787829969.790274:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4117
00000400:00000200:1.0:1787829969.790275:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829969.790289:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829969.790296:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:0.0:1787829969.790299:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829969.790303:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790305:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829969.790307:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790311:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829969.790312:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000020:00000001:0.0:1787829969.790314:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829969.790317:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:0.0:1787829969.790320:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787829969.790323:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000800:00000200:1.0:1787829969.790324:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000020:00000001:0.0:1787829969.790326:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:0.0:1787829969.790330:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000800:00000010:1.0:1787829969.790331:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00000040:0.0:1787829969.790335:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790338:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787829969.790340:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:0.0:1787829969.790343:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000800:00000200:1.0:1787829969.790345:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1787829969.790346:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829969.790351:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000800:00000200:1.1:1787829969.790352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00080000:0.0:1787829969.790356:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000800:00000200:1.1:1787829969.790357:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1787829969.790363:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787829969.790365:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
02000000:00000001:0.0:1787829969.790367:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829969.790370:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829969.790372:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790374:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:1.0:1787829969.790375:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000008:00100000:0.0:1787829969.790376:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829969.790381:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790383:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.790384:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000010:0.0:1787829969.790387:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716f480.
00000020:00000040:0.0:1787829969.790391:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:0.0:1787829969.790393:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829969.790394:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790398:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787829969.790398:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787829969.790400:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829969.790403:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.790405:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
02000000:00000010:0.0:1787829969.790408:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90530400.
00000400:00000010:1.0:1787829969.790409:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:0.0:1787829969.790414:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790416:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:0.0:1787829969.790421:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787829969.790424:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829969.790440:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787829969.790444:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790444:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.790449:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787829969.790453:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84d4029] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790461:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790464:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787829969.790469:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829969.790471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787829969.790475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787829969.790476:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787829969.790477:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787829969.790478:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787829969.790483:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829969.790486:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00080000:0.0:1787829969.790487:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4117
00000100:00000001:1.0:1787829969.790492:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298790573
00000400:00000200:0.0:1787829969.790494:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:0.0:1787829969.790499:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000010:1.0:1787829969.790500:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e631e0.
00000100:00080000:0.0:1787829969.790501:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787829969.790506:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790509:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787829969.790510:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000040:0.0:1787829969.790512:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790515:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829969.790517:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298790573
00000020:00000001:0.0:1787829969.790518:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787829969.790520:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:0.0:1787829969.790523:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:1.0:1787829969.790524:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63840.
00000020:00000040:0.0:1787829969.790525:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829969.790528:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000020:00000001:0.0:1787829969.790529:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:0.0:1787829969.790533:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:0.0:1787829969.790536:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790538:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787829969.790539:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790541:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:0.0:1787829969.790543:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:0.0:1787829969.790545:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:0.0:1787829969.790547:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787829969.790548:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787829969.790550:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829969.790552:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790553:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787829969.790555:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787829969.790558:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790560:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787829969.790563:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd8716d2c0.
00000020:00000040:0.0:1787829969.790565:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:0.0:1787829969.790568:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787829969.790569:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829969.790571:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787829969.790572:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787829969.790573:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787829969.790577:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790577:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790582:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:0.0:1787829969.790584:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd90533000.
00000100:00100000:1.0:1787829969.790588:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969]  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829969.790589:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829969.790598:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:1.0:1787829969.790600:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829969.790603:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829969.790605:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829969.790609:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000100:00000001:0.0:1787829969.790612:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787829969.790612:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:0.0:1787829969.790616:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790617:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1787829969.790625:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c4c777e] to pc [ptlrpcd_rcv+-1]  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829969.790625:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829969.790647:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787829969.790648:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790682:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829969.790684:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790690:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.790694:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787829969.790697:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4127)
00000100:00100000:1.0:1787829969.790702:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790710:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829969.790715:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790727:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829969.790730:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790731:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.790732:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790734:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829969.790736:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829969.790739:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829969.790741:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829969.790743:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829969.790746:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829969.790751:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327397248:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829969.790754:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790756:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.790757:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716de00 x1874671327397248/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790761:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829969.790763:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829969.790765:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829969.790767:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829969.790769:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829969.790770:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829969.790771:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829969.790773:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90532000.
02000000:00000001:1.0:1787829969.790776:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829969.790777:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829969.790778:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716de00.
00000100:00000001:1.0:1787829969.790781:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829969.790781:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790783:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790788:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829969.790789:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790790:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829969.790791:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829969.790792:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829969.790794:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790799:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829969.790800:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.790801:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829969.790802:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829969.790803:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829969.790805:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397376:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829969.790807:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829969.790808:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829969.790809:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.790810:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829969.790812:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829969.790813:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.790816:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829969.790817:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829969.790820:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829969.790822:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397376, portal 10
00000100:00000001:1.0:1787829969.790823:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829969.790824:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419578624 : -116696289972992 : ffff95dd8716ed00)
00000100:00000040:1.0:1787829969.790828:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790832:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829969.790833:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829969.790835:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829969.790836:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327397376, offset 0
00000400:00000200:1.0:1787829969.790839:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829969.790841:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.790844:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829969.790846:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829969.790848:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829969.790849:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829969.790851:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829969.790852:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40876:40876:256:4294967295] 192.168.203.116@tcp LPNI seq info [40876:40876:8:4294967295]
00000400:00000200:1.0:1787829969.790855:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.790859:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829969.790862:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829969.790864:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829969.790866:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829969.790868:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829969.790870:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829969.790872:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787829969.790874:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829969.790876:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829969.790877:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829969.790879:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829969.790881:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829969.790882:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829969.790884:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829969.790886:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790891:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.790894:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790900:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790901:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829969.790904:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829969.790908:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829969.790910:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829969.790912:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298789900
00000100:00000001:1.0:1787829969.790926:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298790573
00000100:00000001:1.0:1787829969.790929:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298789900
00000100:00000001:1.0:1787829969.790931:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298790573
00000100:00000001:1.0:1787829969.790932:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829969.790935:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.790936:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829969.790940:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969]  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829969.790948:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829969.790950:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829969.790951:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829969.790954:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829969.790956:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829969.790958:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829969.790960:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829969.790968:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790974:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829969.790975:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.790977:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.790978:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829969.790982:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790988:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829969.790992:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.790997:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829969.790999:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791000:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.791001:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791002:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829969.791003:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829969.791004:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829969.791006:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829969.791007:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829969.791009:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829969.791014:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397376:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829969.791016:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829969.791018:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791022:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829969.791024:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.791026:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829969.791028:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829969.791029:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829969.791032:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397504:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829969.791035:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829969.791037:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829969.791038:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.791040:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829969.791043:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829969.791045:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.791049:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829969.791052:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829969.791055:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829969.791058:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397504, portal 4
00000100:00000001:1.0:1787829969.791060:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829969.791061:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419580544 : -116696289971072 : ffff95dd8716f480)
00000100:00000040:1.0:1787829969.791066:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791072:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829969.791073:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829969.791075:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829969.791077:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327397504, offset 0
00000400:00000200:1.0:1787829969.791079:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829969.791082:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.791084:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829969.791086:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829969.791088:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829969.791089:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829969.791091:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829969.791093:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40877:40877:256:4294967295] 192.168.203.116@tcp LPNI seq info [40877:40877:8:4294967295]
00000400:00000200:1.0:1787829969.791095:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.791099:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829969.791101:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829969.791103:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829969.791106:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829969.791107:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829969.791110:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829969.791111:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787829969.791114:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829969.791115:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829969.791117:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829969.791119:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829969.791121:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829969.791123:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829969.791125:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829969.791128:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791135:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.791138:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791144:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.791145:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829969.791148:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829969.791151:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829969.791154:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829969.791156:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298789907
00000100:00000001:1.0:1787829969.791159:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298790573
00000100:00000001:1.0:1787829969.791161:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298789907
00000100:00000001:1.0:1787829969.791163:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298790573
00000100:00000001:1.0:1787829969.791165:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829969.791167:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.791169:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829969.791172:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969]  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829969.791179:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829969.791180:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829969.791182:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829969.791184:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829969.791186:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829969.791188:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829969.791191:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829969.791199:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791210:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829969.791211:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791213:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.791214:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829969.791218:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791224:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829969.791228:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791234:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829969.791237:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791238:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.791240:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791241:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829969.791243:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829969.791245:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829969.791248:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829969.791249:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829969.791251:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829969.791256:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397504:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829969.791258:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829969.791260:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791264:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829969.791265:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.791266:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829969.791268:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829969.791269:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829969.791270:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397632:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829969.791273:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829969.791274:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829969.791275:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829969.791276:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829969.791278:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829969.791280:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829969.791283:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829969.791286:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829969.791289:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829969.791291:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397632, portal 4
00000100:00000001:1.0:1787829969.791293:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829969.791294:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377419571904 : -116696289979712 : ffff95dd8716d2c0)
00000100:00000040:1.0:1787829969.791299:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791305:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829969.791307:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829969.791309:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829969.791312:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327397632, offset 0
00000400:00000200:1.0:1787829969.791316:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829969.791319:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.791323:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829969.791325:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829969.791328:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829969.791330:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829969.791332:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829969.791334:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40878:40878:256:4294967295] 192.168.203.116@tcp LPNI seq info [40878:40878:8:4294967295]
00000400:00000200:1.0:1787829969.791337:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829969.791341:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829969.791343:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829969.791346:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829969.791348:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829969.791350:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787829969.791352:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829969.791354:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787829969.791366:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829969.791367:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829969.791370:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829969.791373:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829969.791377:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829969.791378:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829969.791381:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829969.791385:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791393:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.791396:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791402:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.791404:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829969.791407:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829969.791410:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829969.791413:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829969.791415:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298789912
00000100:00000001:1.0:1787829969.791418:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298790573
00000100:00000001:1.0:1787829969.791420:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298789912
00000100:00000001:1.0:1787829969.791422:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298790573
00000100:00000001:1.0:1787829969.791424:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829969.791426:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.791427:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829969.791431:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969]  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829969.791439:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829969.791440:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829969.791442:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829969.791444:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829969.791446:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829969.791448:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829969.791451:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829969.791458:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791463:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829969.791465:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791466:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.791467:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829969.791470:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791476:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829969.791480:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791486:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829969.791488:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791489:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829969.791491:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791492:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829969.791494:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829969.791496:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829969.791499:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829969.791501:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829969.791506:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829969.791513:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd8716d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397632:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829969.791517:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791519:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.791521:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716ed00 x1874671327397376/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791527:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829969.791529:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829969.791531:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829969.791533:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829969.791535:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829969.791536:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829969.791538:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829969.791540:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90531400.
02000000:00000001:1.0:1787829969.791542:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829969.791544:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829969.791545:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716ed00.
00000100:00000001:1.0:1787829969.791547:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829969.791548:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791550:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.791553:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716f480 x1874671327397504/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791559:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829969.791560:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829969.791562:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829969.791564:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829969.791565:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829969.791566:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829969.791568:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829969.791569:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90530400.
02000000:00000001:1.0:1787829969.791572:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829969.791573:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829969.791575:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716f480.
00000100:00000001:1.0:1787829969.791577:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829969.791578:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791579:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829969.791582:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd8716d2c0 x1874671327397632/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829969.791589:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829969.791590:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829969.791591:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829969.791593:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829969.791594:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829969.791595:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829969.791597:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829969.791598:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd90533000.
02000000:00000001:1.0:1787829969.791600:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829969.791601:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829969.791602:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd8716d2c0.
00000100:00000001:1.0:1787829969.791604:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829969.791605:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791607:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829969.791610:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829969.791611:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829969.791613:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829969.791614:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829970.300775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829970.300782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829970.300784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829970.300790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829970.300793:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829970.300797:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829970.300799:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829970.300801:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829970.300804:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829970.492736:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298790573, at 4298790592
00000100:00000040:2.0:1787829970.492740:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298790573, at 4298790592
00000100:00000010:3.0:1787829970.492749:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63840.
00000100:00000010:2.0:1787829970.492751:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e631e0.
00010000:00000001:0.0:1787829970.620765:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829970.620772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829970.620778:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829970.620780:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829970.620784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829970.620787:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829970.620790:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829970.620792:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829970.620795:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829970.620797:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829970.620799:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829970.620801:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829970.621040:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829970.621045:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829970.621050:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829971.324775:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829971.324782:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829971.324784:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829971.324790:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829971.324800:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829971.324804:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829971.324806:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829971.324809:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829971.324812:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829971.644806:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829971.644813:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829971.644819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829971.644821:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829971.644824:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829971.644825:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829971.644988:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829971.644991:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829971.644994:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829972.348781:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829972.348789:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829972.348791:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829972.348796:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829972.348799:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829972.348803:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829972.348805:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829972.348808:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829972.348811:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829972.668789:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829972.668795:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829972.668801:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829972.668802:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829972.668806:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829972.668808:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829972.668829:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829972.668832:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829972.668834:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829973.373134:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829973.373141:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829973.373143:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829973.373149:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829973.373152:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829973.373156:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829973.373159:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829973.373162:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829973.373165:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829973.692738:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829973.692744:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829973.692750:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829973.692751:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829973.692754:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829973.692755:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829973.692811:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829973.692815:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829973.692818:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787829974.396765:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829974.396772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829974.396774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829974.396780:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829974.396783:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829974.396786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829974.396789:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829974.396791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829974.396794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829974.716824:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829974.716830:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829974.716835:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829974.716836:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829974.716839:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829974.716840:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829974.716902:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829974.716905:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829974.716909:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829974.908842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829974.908849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829974.908851:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829974.908854:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829974.908858:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829974.908865:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4122
00000400:00000200:2.0:1787829974.908873:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829974.908879:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829974.908883:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829974.908886:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.908889:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.908891:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.908894:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829974.908898:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829974.908900:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829974.908902:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829974.908905:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829974.908907:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829974.908909:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829974.908911:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.908912:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.908913:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.908916:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829974.908918:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829974.908920:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829974.908922:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829974.908924:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829974.908927:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829974.908928:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.908930:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829974.908931:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829974.908939:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86117c00.
00000020:00000040:2.0:1787829974.908941:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829974.908943:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829974.908944:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.908947:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829974.908949:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829974.908952:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829974.908958:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001400.
00000100:00000001:2.0:1787829974.908962:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829974.908968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829974.908975:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829974.908978:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829974.908981:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c30cb78] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829974.908998:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829974.909000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829974.909002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829974.909003:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829974.909004:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829974.909006:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829974.909012:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4122
00000400:00000200:2.0:1787829974.909017:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829974.909020:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829974.909022:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829974.909026:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909028:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.909030:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909032:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829974.909035:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829974.909036:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829974.909037:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829974.909040:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829974.909042:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829974.909044:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787829974.909044:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000001:1.0:1787829974.909050:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829974.909052:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787829974.909062:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909064:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.909068:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829974.909069:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829974.909070:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829974.909081:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000001:1.0:1787829974.909083:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787829974.909087:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:1.0:1787829974.909088:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909092:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.909094:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787829974.909096:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787829974.909098:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829974.909102:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829974.909103:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787829974.909108:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327397760:192.168.203.116@tcp:250:kworker.0
02000000:00000001:2.0:1787829974.909109:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909113:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829974.909116:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:2.0:1787829974.909119:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787829974.909122:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:2.0:1787829974.909124:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.909126:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909129:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.909130:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:2.0:1787829974.909135:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787829974.909139:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829974.909142:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829974.909143:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787829974.909145:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
00000100:00000001:2.0:1787829974.909146:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829974.909149:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787829974.909150:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829974.909152:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829974.909159:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002800.
00000400:00000200:1.0:1787829974.909165:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:2.0:1787829974.909165:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829974.909172:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000100:00100000:2.0:1787829974.909174:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787829974.909181:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000001:2.0:1787829974.909184:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787829974.909189:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397760, portal 25
00000100:00000001:1.0:1787829974.909193:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829974.909194:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402444800 : -116696307106816 : ffff95dd86117c00)
00000100:00000001:2.0:1787829974.909199:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829974.909203:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787829974.909208:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.909222:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787829974.909227:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829974.909229:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829974.909231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1787829974.909234:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00080000:2.0:1787829974.909236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829974.909239:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:1.0:1787829974.909240:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327397760, offset 0
00000100:00080000:2.0:1787829974.909242:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829974.909246:0:37:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787829974.909250:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00080000:2.0:1787829974.909252:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4122
00000400:00000200:2.0:1787829974.909258:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.909261:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829974.909264:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829974.909266:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829974.909268:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909269:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.909271:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909273:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829974.909275:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829974.909276:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829974.909277:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787829974.909280:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000020:00000040:2.0:1787829974.909281:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000400:00000200:1.0:1787829974.909289:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829974.909293:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:2.0:1787829974.909293:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829974.909299:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000200:1.0:1787829974.909300:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829974.909307:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000040:2.0:1787829974.909307:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909312:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787829974.909316:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40879:40879:256:4294967295] 192.168.203.116@tcp LPNI seq info [40879:40879:8:4294967295]
00000100:00000040:2.0:1787829974.909318:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909324:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829974.909328:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000020:00000040:2.0:1787829974.909329:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000400:00000200:1.0:1787829974.909341:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1787829974.909346:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909352:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829974.909353:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:2.0:1787829974.909356:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:1.0:1787829974.909361:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
02000000:00000001:2.0:1787829974.909362:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829974.909367:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909369:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:1.0:1787829974.909373:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000008:00100000:2.0:1787829974.909373:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829974.909379:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787829974.909380:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1787829974.909382:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829974.909388:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000800:00000200:1.1:1787829974.909389:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000020:00000040:2.0:1787829974.909393:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
00000800:00000200:1.1:1787829974.909395:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
02000000:00000001:2.0:1787829974.909397:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829974.909399:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909402:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829974.909403:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829974.909404:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829974.909407:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002400.
00000100:00000001:2.0:1787829974.909409:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829974.909414:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:1.0:1787829974.909419:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.909424:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000100:00000001:2.0:1787829974.909424:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829974.909426:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829974.909431:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829974.909433:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829974.909444:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787829974.909445:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829974.909449:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787829974.909450:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829974.909455:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00080000:2.0:1787829974.909456:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829974.909459:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787829974.909461:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1787829974.909462:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829974.909466:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1787829974.909469:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787829974.909471:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4122
00000400:00000200:2.0:1787829974.909479:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829974.909486:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00000001:1.0:1787829974.909489:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1787829974.909490:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829974.909497:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000040:1.0:1787829974.909498:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829974.909500:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.909505:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909510:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829974.909516:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829974.909516:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829974.909519:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000100:00000001:1.0:1787829974.909521:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:2.0:1787829974.909523:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829974.909526:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000100:00000001:1.0:1787829974.909529:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829974.909529:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829974.909533:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:1.0:1787829974.909538:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:2.0:1787829974.909540:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:1.0:1787829974.909543:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298795692
00000100:00000001:2.0:1787829974.909545:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829974.909550:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000010:1.0:1787829974.909551:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63060.
00000100:00000001:2.0:1787829974.909553:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829974.909557:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00000001:1.0:1787829974.909560:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00080000:2.0:1787829974.909561:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829974.909566:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829974.909568:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829974.909569:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298795692
02000000:00000001:2.0:1787829974.909572:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:1.0:1787829974.909575:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63240.
02000000:00000001:2.0:1787829974.909576:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909580:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787829974.909581:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000008:00100000:2.0:1787829974.909584:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787829974.909587:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1787829974.909588:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.909592:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909593:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.909597:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829974.909604:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974]  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787829974.909604:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829974.909610:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829974.909630:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829974.909631:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829974.909632:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829974.909633:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829974.909635:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829974.909638:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87000400.
00000100:00000001:2.0:1787829974.909640:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829974.909644:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829974.909649:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829974.909650:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829974.909653:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829974.909694:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829974.909697:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4132)
00000400:00000001:1.0:1787829974.909849:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829974.909851:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829974.909853:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829974.909857:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829974.909858:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829974.909861:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829974.909866:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829974.909875:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.909882:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829974.909884:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.909887:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.909889:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829974.909892:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.909899:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829974.909904:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910029:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829974.910033:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910034:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910036:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910038:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829974.910039:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829974.910042:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829974.910046:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829974.910049:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829974.910054:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829974.910061:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86117c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327397760:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829974.910068:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910070:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.910074:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86117c00 x1874671327397760/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910080:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829974.910083:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829974.910086:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829974.910089:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829974.910091:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829974.910092:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829974.910094:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829974.910097:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001400.
02000000:00000001:1.0:1787829974.910100:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829974.910102:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829974.910103:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86117c00.
00000100:00000001:1.0:1787829974.910106:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829974.910107:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910109:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910116:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829974.910118:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910120:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829974.910122:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829974.910123:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829974.910127:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910133:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829974.910135:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.910137:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829974.910139:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829974.910141:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829974.910144:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397888:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829974.910147:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829974.910150:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829974.910152:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.910154:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829974.910157:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829974.910160:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829974.910164:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829974.910167:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829974.910170:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829974.910174:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327397888, portal 10
00000100:00000001:1.0:1787829974.910175:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829974.910177:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:1.0:1787829974.910183:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910189:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829974.910191:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829974.910194:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829974.910196:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327397888, offset 0
00000400:00000200:1.0:1787829974.910200:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829974.910204:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.910208:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829974.910212:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829974.910214:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829974.910216:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829974.910219:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829974.910222:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40880:40880:256:4294967295] 192.168.203.116@tcp LPNI seq info [40880:40880:8:4294967295]
00000400:00000200:1.0:1787829974.910226:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.910232:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829974.910243:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.910246:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829974.910251:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829974.910254:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829974.910258:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829974.910261:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829974.910264:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.910266:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829974.910269:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829974.910273:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829974.910276:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829974.910278:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829974.910280:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829974.910284:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910292:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.910295:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910300:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910302:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829974.910305:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829974.910308:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829974.910312:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829974.910314:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298795019
00000100:00000001:1.0:1787829974.910316:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298795692
00000100:00000001:1.0:1787829974.910318:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298795019
00000100:00000001:1.0:1787829974.910320:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298795692
00000100:00000001:1.0:1787829974.910322:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829974.910324:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910326:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829974.910329:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974]  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829974.910337:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829974.910339:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829974.910340:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829974.910343:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829974.910344:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829974.910347:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829974.910350:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829974.910358:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910365:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829974.910366:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910368:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910369:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829974.910372:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910378:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829974.910382:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910388:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829974.910391:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910392:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910394:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910395:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829974.910397:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829974.910400:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829974.910403:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829974.910405:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829974.910409:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829974.910417:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327397888:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829974.910420:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829974.910423:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910435:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829974.910437:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.910439:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829974.910441:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829974.910443:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829974.910445:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398016:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829974.910448:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829974.910450:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829974.910451:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.910453:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829974.910457:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829974.910459:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829974.910462:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829974.910465:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829974.910468:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829974.910471:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398016, portal 4
00000100:00000001:1.0:1787829974.910472:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829974.910474:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:1.0:1787829974.910479:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910484:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829974.910486:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829974.910488:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829974.910491:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327398016, offset 0
00000400:00000200:1.0:1787829974.910494:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829974.910498:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.910501:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829974.910504:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829974.910506:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829974.910508:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829974.910510:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829974.910513:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40881:40881:256:4294967295] 192.168.203.116@tcp LPNI seq info [40881:40881:8:4294967295]
00000400:00000200:1.0:1787829974.910518:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.910523:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829974.910527:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.910530:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829974.910534:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829974.910537:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829974.910541:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829974.910543:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829974.910547:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.910548:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829974.910551:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829974.910554:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829974.910556:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829974.910558:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829974.910560:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829974.910563:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910571:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.910574:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910580:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910581:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829974.910584:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829974.910587:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829974.910590:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829974.910592:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298795026
00000100:00000001:1.0:1787829974.910594:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298795692
00000100:00000001:1.0:1787829974.910596:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298795026
00000100:00000001:1.0:1787829974.910598:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298795692
00000100:00000001:1.0:1787829974.910600:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829974.910602:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910604:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829974.910607:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974]  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829974.910630:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829974.910631:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829974.910633:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829974.910636:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829974.910637:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829974.910639:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829974.910643:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829974.910650:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910679:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829974.910680:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910682:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910683:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829974.910687:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910693:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829974.910704:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910711:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829974.910713:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910714:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910716:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910717:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829974.910719:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829974.910721:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829974.910724:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829974.910726:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829974.910729:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829974.910736:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398016:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829974.910739:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829974.910743:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910754:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829974.910756:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.910758:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829974.910760:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829974.910762:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829974.910765:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398144:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829974.910767:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829974.910769:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829974.910770:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829974.910772:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829974.910775:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829974.910777:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829974.910780:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829974.910782:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829974.910785:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829974.910788:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398144, portal 4
00000100:00000001:1.0:1787829974.910790:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829974.910791:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:1.0:1787829974.910796:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910802:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829974.910805:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829974.910807:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829974.910809:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327398144, offset 0
00000400:00000200:1.0:1787829974.910813:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829974.910816:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.910820:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829974.910823:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829974.910825:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829974.910827:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829974.910829:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829974.910832:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40882:40882:256:4294967295] 192.168.203.116@tcp LPNI seq info [40882:40882:8:4294967295]
00000400:00000200:1.0:1787829974.910836:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829974.910841:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829974.910845:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.910848:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829974.910852:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829974.910855:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787829974.910859:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829974.910861:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787829974.910864:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829974.910866:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829974.910869:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829974.910872:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829974.910875:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829974.910877:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829974.910879:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829974.910882:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910889:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.910892:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787829999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910898:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910900:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829974.910903:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829974.910905:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829974.910909:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829974.910919:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298795032
00000100:00000001:1.0:1787829974.910921:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298795692
00000100:00000001:1.0:1787829974.910923:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298795032
00000100:00000001:1.0:1787829974.910924:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298795692
00000100:00000001:1.0:1787829974.910926:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829974.910928:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.910929:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829974.910933:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974]  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829974.910939:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829974.910940:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829974.910942:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829974.910944:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829974.910945:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829974.910947:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829974.910949:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829974.910956:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910961:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829974.910962:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910964:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910965:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829974.910968:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910974:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829974.910977:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.910982:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829974.910984:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910986:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829974.910987:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.910989:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829974.910990:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829974.910993:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829974.910996:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829974.910998:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829974.911001:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829974.911008:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398144:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829974.911011:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.911013:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.911015:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327397888/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.911021:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829974.911023:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829974.911025:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829974.911027:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829974.911029:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829974.911030:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829974.911033:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829974.911034:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002800.
02000000:00000001:1.0:1787829974.911037:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829974.911038:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829974.911039:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:1.0:1787829974.911041:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829974.911043:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.911045:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.911047:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327398016/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.911053:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829974.911055:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829974.911056:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829974.911058:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829974.911059:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829974.911061:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829974.911062:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829974.911064:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002400.
02000000:00000001:1.0:1787829974.911066:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829974.911067:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829974.911068:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:1.0:1787829974.911070:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829974.911071:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.911072:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829974.911075:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327398144/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829974.911081:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829974.911082:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829974.911083:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829974.911085:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829974.911086:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829974.911087:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829974.911089:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829974.911090:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87000400.
02000000:00000001:1.0:1787829974.911092:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829974.911093:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829974.911094:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:1.0:1787829974.911097:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829974.911098:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.911100:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829974.911104:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829974.911105:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829974.911107:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829974.911110:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787829975.420755:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787829975.420762:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787829975.420764:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787829975.420769:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:0.0:1787829975.420772:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787829975.420776:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829975.420778:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787829975.420780:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787829975.420783:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829975.612729:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298795692, at 4298795712
00000100:00000010:3.0:1787829975.612744:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63240.
00000100:00000040:2.0:1787829975.612921:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298795692, at 4298795712
00000100:00000010:2.0:1787829975.612927:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63060.
00010000:00000001:0.0:1787829975.740819:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829975.740827:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829975.740829:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829975.740833:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740837:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829975.740839:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740841:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740843:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740847:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829975.740848:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829975.740849:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829975.740851:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740852:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829975.740853:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740855:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740857:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829975.740859:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829975.740861:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829975.740931:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829975.740933:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829975.740936:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829976.444862:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829976.444870:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829976.444873:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829976.444880:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829976.444885:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829976.444889:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829976.444892:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829976.444895:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829976.444899:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829976.765730:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829976.765739:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829976.765746:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829976.765747:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829976.765751:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829976.765752:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829976.765850:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829976.765854:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829976.765857:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829977.469069:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829977.469078:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829977.469081:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829977.469087:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829977.469091:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829977.469096:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829977.469100:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829977.469103:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829977.469106:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829977.788791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829977.788800:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829977.788807:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829977.788809:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829977.788813:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829977.788814:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829977.789381:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829977.789386:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829977.789390:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829978.492803:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829978.492820:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829978.492826:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829978.492832:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829978.492836:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829978.492840:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829978.492849:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829978.492851:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829978.492855:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829978.812784:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829978.812790:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829978.812796:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829978.812798:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829978.812800:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829978.812801:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829978.812826:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829978.812828:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829978.812831:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829979.516779:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829979.516785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829979.516787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829979.516792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829979.516795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829979.516798:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829979.516801:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829979.516803:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829979.516806:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829979.837676:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829979.837682:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829979.837688:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829979.837689:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829979.837692:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829979.837694:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829979.838119:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829979.838122:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829979.838126:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787829980.028808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829980.028818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787829980.028820:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829980.028824:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829980.028828:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829980.028835:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4127
00000400:00000200:3.0:1787829980.028844:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829980.028851:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829980.028854:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829980.028857:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.028860:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.028862:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.028865:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829980.028869:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829980.028871:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:3.0:1787829980.028873:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829980.028876:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:3.0:1787829980.028878:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:3.0:1787829980.028880:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:3.0:1787829980.028883:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.028884:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.028886:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.028888:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829980.028891:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:3.0:1787829980.028894:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829980.028897:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829980.028899:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829980.028902:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829980.028905:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.028907:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787829980.028909:0:36:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829980.028917:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed0f00.
00000020:00000040:3.0:1787829980.028920:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:3.0:1787829980.028924:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829980.028926:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.028928:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829980.028931:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829980.028935:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829980.028940:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3400.
00000100:00000001:3.0:1787829980.028946:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829980.028953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787829980.028960:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829980.028964:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829980.028967:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf76b35] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787829980.028984:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829980.028986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829980.028988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787829980.028989:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829980.028990:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829980.028992:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829980.028993:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4127
00000400:00000200:3.0:1787829980.028996:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829980.028997:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829980.028999:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829980.029001:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029002:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.029003:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029004:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829980.029006:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829980.029007:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:3.0:1787829980.029008:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829980.029009:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:3.0:1787829980.029011:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:3.0:1787829980.029013:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:3.0:1787829980.029015:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029017:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.029018:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029019:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000100:00000001:1.0:1787829980.029020:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829980.029024:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829980.029026:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787829980.029029:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00000040:1.0:1787829980.029034:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787829980.029035:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029040:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029046:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787829980.029046:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787829980.029049:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.029052:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829980.029055:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829980.029057:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829980.029057:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829980.029062:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829980.029066:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327398272:192.168.203.116@tcp:250:kworker.0
00000100:00000001:3.0:1787829980.029066:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829980.029071:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829980.029073:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029075:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787829980.029081:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3c00.
00000100:00000001:1.0:1787829980.029082:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1787829980.029084:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:3.0:1787829980.029087:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787829980.029088:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787829980.029088:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029092:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787829980.029093:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829980.029093:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829980.029096:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829980.029099:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3c00.
00000100:00000001:3.0:1787829980.029102:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.029105:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1787829980.029106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1787829980.029112:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
00000100:00000001:3.0:1787829980.029113:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829980.029115:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.029117:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829980.029119:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8bcc137] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787829980.029126:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:3.0:1787829980.029126:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829980.029128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829980.029131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1787829980.029132:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829980.029133:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787829980.029134:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829980.029136:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4127
00000400:00000200:3.0:1787829980.029139:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829980.029141:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829980.029142:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787829980.029144:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029145:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.029148:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029149:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000010:1.0:1787829980.029150:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000020:00000001:3.0:1787829980.029151:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829980.029153:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:3.0:1787829980.029155:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829980.029156:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:3.0:1787829980.029158:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:3.0:1787829980.029159:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000400:00000010:1.0:1787829980.029160:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000040:3.0:1787829980.029161:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029165:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.029166:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029167:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:3.0:1787829980.029169:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00000200:1.0:1787829980.029170:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398272, portal 25
00000100:00080000:3.0:1787829980.029171:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029175:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029176:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1787829980.029176:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787829980.029179:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829980.029180:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029181:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829980.029182:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829980.029183:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029184:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029188:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400057600 : -116696309494016 : ffff95dd85ed0f00)
00000100:00000010:3.0:1787829980.029188:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed3840.
00000020:00000040:3.0:1787829980.029192:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:3.0:1787829980.029194:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829980.029195:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029197:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1787829980.029199:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787829980.029199:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829980.029200:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829980.029205:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a2800.
00000100:00000001:3.0:1787829980.029208:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829980.029214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787829980.029215:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829980.029223:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029223:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829980.029225:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787829980.029229:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000040:3.0:1787829980.029230:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000062f78266] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829980.029236:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327398272, offset 0
00000100:00000001:3.0:1787829980.029237:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787829980.029240:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787829980.029243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787829980.029244:0:36:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787829980.029245:0:36:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787829980.029250:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029250:0:36:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787829980.029254:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4127
00000400:00000200:3.0:1787829980.029257:0:36:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:3.0:1787829980.029258:0:36:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:3.0:1787829980.029260:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787829980.029263:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000040:3.0:1787829980.029264:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029266:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.029268:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829980.029271:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000100:00000001:3.0:1787829980.029271:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:3.0:1787829980.029275:0:36:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787829980.029277:0:36:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000400:00000200:1.0:1787829980.029278:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:3.0:1787829980.029279:0:36:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787829980.029282:0:36:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000400:00000200:1.0:1787829980.029285:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000020:00000001:3.0:1787829980.029285:0:36:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:3.0:1787829980.029289:0:36:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000400:00000200:1.0:1787829980.029290:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1787829980.029293:0:36:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000400:00000200:1.0:1787829980.029296:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029297:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787829980.029300:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029303:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000400:00000200:1.0:1787829980.029304:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40883:40883:256:4294967295] 192.168.203.116@tcp LPNI seq info [40883:40883:8:4294967295]
00000020:00000040:3.0:1787829980.029305:0:36:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:3.0:1787829980.029309:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:3.0:1787829980.029313:0:36:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787829980.029316:0:36:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787829980.029317:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
02000000:00000001:3.0:1787829980.029319:0:36:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787829980.029322:0:36:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029324:0:36:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787829980.029326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787829980.029330:0:36:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029334:0:36:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829980.029335:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000100:00000010:3.0:1787829980.029338:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd85ed1e00.
00000020:00000040:3.0:1787829980.029342:0:36:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
00000800:00000200:1.0:1787829980.029345:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
02000000:00000001:3.0:1787829980.029345:0:36:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787829980.029347:0:36:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787829980.029348:0:36:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787829980.029349:0:36:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787829980.029351:0:36:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787829980.029356:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd861a3000.
00000800:00000010:1.0:1787829980.029360:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000100:00000001:3.0:1787829980.029361:0:36:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787829980.029368:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1787829980.029371:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829980.029377:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000100:00000001:3.0:1787829980.029380:0:36:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787829980.029384:0:36:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787829980.029387:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000100:00000040:3.0:1787829980.029391:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b795630b] to pc [ptlrpcd_rcv+-1]  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.1:1787829980.029393:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787829980.029403:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.029406:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829980.029412:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829980.029418:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787829980.029424:0:36:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829980.029425:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000100:00000040:3.0:1787829980.029429:0:36:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4137)
00000400:00000010:1.0:1787829980.029433:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829980.029438:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829980.029442:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029454:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.029461:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029469:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029471:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829980.029476:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829980.029481:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829980.029484:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298800812
00000100:00000010:1.0:1787829980.029489:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63f60.
00000100:00000001:1.0:1787829980.029495:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829980.029499:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298800812
00000100:00000010:1.0:1787829980.029502:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63ae0.
00000100:00000001:1.0:1787829980.029505:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829980.029514:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829980.029517:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029520:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829980.029525:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829980/real 1787829980]  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829980.029534:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829980.029536:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829980.029538:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829980.029542:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829980.029544:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829980.029547:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829980.029550:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829980.029563:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029570:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829980.029571:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029574:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.029576:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829980.029594:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029602:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829980.029609:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029616:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829980.029619:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029621:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.029631:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029634:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829980.029636:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829980.029639:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829980.029643:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829980.029646:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829980.029654:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829980.029688:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327398272:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829980.029693:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029695:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.029698:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed0f00 x1874671327398272/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029704:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829980.029707:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829980.029709:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829980.029712:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829980.029713:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829980.029715:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829980.029717:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829980.029720:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3400.
02000000:00000001:1.0:1787829980.029723:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829980.029725:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829980.029727:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed0f00.
00000100:00000001:1.0:1787829980.029730:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829980.029731:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029733:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029740:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829980.029742:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029744:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829980.029745:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829980.029747:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829980.029751:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029763:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829980.029764:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.029767:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829980.029768:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829980.029771:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829980.029773:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398400:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829980.029776:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829980.029778:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829980.029780:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.029782:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829980.029785:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829980.029787:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829980.029790:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829980.029792:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829980.029795:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829980.029798:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398400, portal 10
00000100:00000001:1.0:1787829980.029800:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829980.029802:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400069120 : -116696309482496 : ffff95dd85ed3c00)
00000100:00000040:1.0:1787829980.029808:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029813:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829980.029816:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829980.029817:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829980.029820:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327398400, offset 0
00000400:00000200:1.0:1787829980.029823:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829980.029827:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829980.029837:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829980.029840:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829980.029842:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829980.029845:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829980.029847:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829980.029850:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40884:40884:256:4294967295] 192.168.203.116@tcp LPNI seq info [40884:40884:8:4294967295]
00000400:00000200:1.0:1787829980.029854:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829980.029860:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829980.029864:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.029868:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829980.029871:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829980.029874:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787829980.029878:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829980.029881:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787829980.029884:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.029886:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829980.029889:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829980.029892:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829980.029895:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829980.029896:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829980.029898:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829980.029902:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029910:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.029913:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029919:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029921:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829980.029924:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829980.029927:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829980.029930:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829980.029932:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298800139
00000100:00000001:1.0:1787829980.029935:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298800812
00000100:00000001:1.0:1787829980.029937:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298800139
00000100:00000001:1.0:1787829980.029939:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298800812
00000100:00000001:1.0:1787829980.029940:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829980.029943:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.029944:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829980.029948:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829980/real 1787829980]  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829980.029955:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829980.029956:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829980.029958:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829980.029960:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829980.029962:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829980.029964:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829980.029966:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829980.029973:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029980:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829980.029981:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.029982:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.029984:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829980.029987:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.029992:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829980.029996:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030002:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829980.030004:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030006:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.030007:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030009:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829980.030010:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829980.030013:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829980.030017:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829980.030019:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829980.030023:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829980.030029:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398400:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829980.030033:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829980.030036:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030042:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829980.030043:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.030045:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829980.030047:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829980.030049:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829980.030051:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398528:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829980.030054:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829980.030055:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829980.030057:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.030059:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829980.030062:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829980.030064:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829980.030067:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829980.030069:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829980.030072:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829980.030075:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398528, portal 4
00000100:00000001:1.0:1787829980.030077:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829980.030078:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400068160 : -116696309483456 : ffff95dd85ed3840)
00000100:00000040:1.0:1787829980.030083:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030089:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829980.030091:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829980.030093:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829980.030096:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327398528, offset 0
00000400:00000200:1.0:1787829980.030100:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829980.030104:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829980.030107:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829980.030109:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829980.030112:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829980.030114:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829980.030116:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829980.030119:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40885:40885:256:4294967295] 192.168.203.116@tcp LPNI seq info [40885:40885:8:4294967295]
00000400:00000200:1.0:1787829980.030123:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829980.030130:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829980.030134:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.030137:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829980.030140:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829980.030143:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787829980.030147:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829980.030150:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787829980.030153:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.030155:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829980.030158:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829980.030161:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829980.030164:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829980.030166:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829980.030168:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829980.030172:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030178:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.030181:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030188:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.030189:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829980.030192:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829980.030195:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829980.030198:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829980.030200:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298800146
00000100:00000001:1.0:1787829980.030202:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298800812
00000100:00000001:1.0:1787829980.030204:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298800146
00000100:00000001:1.0:1787829980.030206:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298800812
00000100:00000001:1.0:1787829980.030208:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829980.030210:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.030211:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829980.030216:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829980/real 1787829980]  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829980.030224:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829980.030225:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829980.030227:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829980.030229:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829980.030230:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829980.030232:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829980.030235:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829980.030242:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030249:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829980.030250:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030251:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.030253:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829980.030256:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030262:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829980.030266:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030274:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829980.030276:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030277:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.030279:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030280:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829980.030282:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829980.030284:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829980.030287:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829980.030289:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829980.030293:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829980.030300:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398528:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829980.030304:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829980.030308:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030314:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829980.030316:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.030318:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829980.030319:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829980.030321:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829980.030324:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398656:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829980.030327:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829980.030329:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829980.030330:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829980.030332:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829980.030335:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829980.030338:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829980.030341:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829980.030343:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829980.030346:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829980.030349:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398656, portal 4
00000100:00000001:1.0:1787829980.030351:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829980.030352:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377400061440 : -116696309490176 : ffff95dd85ed1e00)
00000100:00000040:1.0:1787829980.030357:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030362:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829980.030364:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829980.030367:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829980.030369:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327398656, offset 0
00000400:00000200:1.0:1787829980.030372:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829980.030375:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829980.030378:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829980.030381:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829980.030383:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829980.030385:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829980.030387:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829980.030390:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40886:40886:256:4294967295] 192.168.203.116@tcp LPNI seq info [40886:40886:8:4294967295]
00000400:00000200:1.0:1787829980.030394:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829980.030400:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829980.030404:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.030407:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829980.030410:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829980.030413:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787829980.030417:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829980.030419:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787829980.030423:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829980.030424:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829980.030427:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829980.030430:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829980.030433:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829980.030434:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829980.030437:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829980.030440:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030447:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.030450:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030456:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.030457:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829980.030460:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829980.030462:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829980.030466:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829980.030467:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298800151
00000100:00000001:1.0:1787829980.030470:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298800812
00000100:00000001:1.0:1787829980.030472:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298800151
00000100:00000001:1.0:1787829980.030473:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298800812
00000100:00000001:1.0:1787829980.030475:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829980.030477:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.030478:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829980.030482:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829980/real 1787829980]  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829980.030489:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829980.030490:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829980.030491:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829980.030493:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829980.030495:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829980.030496:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829980.030499:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829980.030506:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030516:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829980.030517:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030519:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.030520:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829980.030523:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030529:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829980.030532:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030538:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829980.030540:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030541:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829980.030542:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030544:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829980.030545:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829980.030547:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829980.030550:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829980.030552:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829980.030555:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829980.030561:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd85ed1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398656:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829980.030564:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030566:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.030569:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3c00 x1874671327398400/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030574:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829980.030576:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829980.030590:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829980.030593:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829980.030594:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829980.030596:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829980.030598:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829980.030599:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3c00.
02000000:00000001:1.0:1787829980.030601:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829980.030603:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829980.030604:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3c00.
00000100:00000001:1.0:1787829980.030607:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829980.030608:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030609:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.030613:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed3840 x1874671327398528/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030619:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829980.030620:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829980.030622:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829980.030624:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829980.030625:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829980.030626:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829980.030628:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829980.030629:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a2800.
02000000:00000001:1.0:1787829980.030631:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829980.030632:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829980.030633:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed3840.
00000100:00000001:1.0:1787829980.030634:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829980.030636:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030637:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829980.030640:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd85ed1e00 x1874671327398656/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830005 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829980.030644:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829980.030645:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829980.030646:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5000.
02000000:00000001:1.0:1787829980.030647:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829980.030648:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829980.030649:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829980.030650:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829980.030651:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd861a3000.
02000000:00000001:1.0:1787829980.030652:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829980.030653:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829980.030654:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd85ed1e00.
00000100:00000001:1.0:1787829980.030671:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829980.030672:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030673:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829980.030676:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829980.030677:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829980.030678:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829980.030679:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829980.540813:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829980.540820:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829980.540822:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829980.540827:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829980.540830:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829980.540834:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829980.540837:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829980.540840:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829980.540842:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787829980.732743:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298800812, at 4298800832
00000100:00000010:3.0:1787829980.732759:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63ae0.
00000100:00000040:2.0:1787829980.732833:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298800812, at 4298800832
00000100:00000010:2.0:1787829980.732837:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63f60.
00010000:00000001:0.0:1787829980.860779:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829980.860785:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829980.860791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829980.860792:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829980.860795:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829980.860798:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829980.860801:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829980.860803:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829980.860806:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829980.860808:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829980.860810:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829980.860812:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829980.860903:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829980.860906:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829980.860908:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829981.564837:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829981.565239:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829981.565241:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829981.565255:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829981.565259:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829981.565266:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829981.565269:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829981.565272:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829981.565275:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829981.885609:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829981.885615:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829981.885621:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829981.885622:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829981.885625:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829981.885626:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829981.885642:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829981.885645:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829981.885648:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829982.588749:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829982.588754:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829982.588755:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829982.588759:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829982.588762:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829982.588765:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829982.588767:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829982.588768:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829982.588770:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829982.908775:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829982.908782:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829982.908787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829982.908789:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829982.908791:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829982.908793:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829982.909046:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829982.909050:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829982.909053:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829983.612778:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829983.612785:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829983.612787:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829983.612792:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829983.612795:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829983.612800:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829983.612802:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829983.612805:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829983.612808:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829983.932738:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829983.932745:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829983.932752:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829983.932753:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829983.932757:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829983.932759:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829983.932840:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829983.932843:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829983.932846:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829984.636764:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829984.636772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829984.636774:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829984.636779:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829984.636782:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829984.636786:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829984.636788:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829984.636791:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829984.636794:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829984.956787:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829984.956794:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829984.956799:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829984.956800:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829984.956802:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829984.956803:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829984.956843:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829984.956846:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829984.956849:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787829985.148813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 81801ac7-a7ac-4663-9aee-7c551b573152->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829985.148824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787829985.148826:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829985.148830:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829985.148834:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829985.148841:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.116@tcp: connect to NID 192.168.203.116@tcp last attempt 4132
00000400:00000200:2.0:1787829985.148850:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829985.148856:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829985.148859:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.116@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829985.148862:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.148865:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.148867:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.148869:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829985.148873:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829985.148875:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a2760
00000020:00000001:2.0:1787829985.148878:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829985.148881:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90723000 refcount=3
00000020:00000001:2.0:1787829985.148883:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000020:00000001:2.0:1787829985.148886:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576554496 : -116696132997120 : ffff95dd90723000)
00000100:00000040:2.0:1787829985.148889:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.148891:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.148893:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.148895:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829985.148898:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90723000 : new refcount 2
00000100:00080000:2.0:1787829985.148901:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.116@tcp: import ffff95dd82e32000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829985.148903:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829985.148906:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829985.148910:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829985.148913:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.148915:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787829985.148918:0:37:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829985.148927:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861143c0.
00000020:00000040:2.0:1787829985.148930:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd82e32000 refcount=4 obd=MGC192.168.203.116@tcp
02000000:00000001:2.0:1787829985.148934:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829985.148936:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.148939:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829985.148942:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829985.148945:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829985.148952:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001800.
00000100:00000001:2.0:1787829985.148958:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829985.148966:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829985.148976:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829985.148980:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829985.148985:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff86c710] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829985.149012:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829985.149015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829985.149019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787829985.149020:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829985.149022:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829985.149026:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829985.149028:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4132
00000400:00000200:2.0:1787829985.149040:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829985.149043:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829985.149046:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829985.149049:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149051:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.149053:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149055:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829985.149057:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829985.149058:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27ad
00000020:00000001:2.0:1787829985.149060:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829985.149062:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd90721800 refcount=5
00000020:00000001:2.0:1787829985.149064:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000020:00000001:2.0:1787829985.149066:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377576548352 : -116696133003264 : ffff95dd90721800)
00000100:00000040:2.0:1787829985.149068:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149070:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.149072:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149074:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829985.149076:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd90721800 : new refcount 4
00000100:00080000:2.0:1787829985.149086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff95dd94719800: import ffff95dd9273d000 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149089:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829985.149091:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829985.149096:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829985.149098:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149099:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829985.149103:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829985.149106:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149107:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829985.149113:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86116940.
00000020:00000040:2.0:1787829985.149115:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd9273d000 refcount=72 obd=lustre-MDT0000-mdc-ffff95dd94719800
02000000:00000001:2.0:1787829985.149117:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829985.149119:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149120:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829985.149122:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829985.149123:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829985.149127:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87003000.
00000100:00000001:2.0:1787829985.149130:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829985.149135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829985.149143:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829985.149145:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829985.149149:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4ee3f01] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829985.149157:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829985.149159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829985.149162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1787829985.149163:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829985.149176:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829985.149178:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829985.149181:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4132
00000400:00000200:2.0:1787829985.149185:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829985.149188:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829985.149190:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829985.149193:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149195:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.149197:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149199:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829985.149201:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829985.149203:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27bb
00000020:00000001:2.0:1787829985.149205:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829985.149206:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a0000 refcount=4
00000020:00000001:2.0:1787829985.149209:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000020:00000001:2.0:1787829985.149211:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609572352 : -116696099979264 : ffff95dd926a0000)
00000100:00000040:2.0:1787829985.149219:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149221:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.149223:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149225:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829985.149228:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a0000 : new refcount 3
00000100:00080000:2.0:1787829985.149230:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff95dd94719800: import ffff95dd90720800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149232:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829985.149234:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829985.149237:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829985.149239:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149241:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829985.149243:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829985.149245:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149247:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829985.149251:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd861152c0.
00000020:00000040:2.0:1787829985.149253:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd90720800 refcount=3 obd=lustre-OST0000-osc-ffff95dd94719800
02000000:00000001:2.0:1787829985.149255:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829985.149256:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149258:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829985.149259:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829985.149261:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829985.149264:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87002000.
00000100:00000001:2.0:1787829985.149267:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829985.149272:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829985.149279:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829985.149282:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829985.149286:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093709bb0] to pc [ptlrpcd_rcv+-1]  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829985.149293:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787829985.149296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 487ec2ab-0772-4b76-b5b3-7e1d3e9f5308->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787829985.149299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787829985.149300:0:37:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787829985.149301:0:37:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787829985.149303:0:37:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787829985.149306:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: connect to NID 192.168.203.116@tcp last attempt 4132
00000400:00000200:2.0:1787829985.149309:0:37:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:2.0:1787829985.149311:0:37:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.116@tcp is discovered: rc = 1
00000100:00080000:2.0:1787829985.149314:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787829985.149316:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149318:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.149320:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149322:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000001:2.0:1787829985.149324:0:37:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787829985.149326:0:37:0:(genops.c:804:class_conn2export()) looking for export cookie 0x8a53771c955a27c2
00000020:00000001:2.0:1787829985.149328:0:37:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787829985.149329:0:37:0:(lustre_handles.c:148:class_handle2object()) GET export ffff95dd926a6000 refcount=4
00000020:00000001:2.0:1787829985.149331:0:37:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000020:00000001:2.0:1787829985.149333:0:37:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446627377609596928 : -116696099954688 : ffff95dd926a6000)
00000100:00000040:2.0:1787829985.149335:0:37:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff95dd982ac6c0 refcount 11 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149337:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787829985.149339:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff95dd982ac6c0 refcount 12 to 192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149341:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446627377706092224 : -116696003459392 : ffff95dd982ac6c0)
00000020:00000040:2.0:1787829985.149343:0:37:0:(genops.c:879:class_export_put()) PUTting export ffff95dd926a6000 : new refcount 3
00000100:00080000:2.0:1787829985.149346:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff95dd94719800: import ffff95dd91f74800 using connection 192.168.203.116@tcp/192.168.203.116@tcp
00000100:00000001:2.0:1787829985.149348:0:37:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787829985.149350:0:37:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787829985.149352:0:37:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787829985.149354:0:37:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149356:0:37:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787829985.149358:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787829985.149360:0:37:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149362:0:37:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787829985.149365:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff95dd86114780.
00000020:00000040:2.0:1787829985.149367:0:37:0:(genops.c:1210:class_import_get()) import ffff95dd91f74800 refcount=4 obd=lustre-OST0001-osc-ffff95dd94719800
02000000:00000001:2.0:1787829985.149369:0:37:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787829985.149371:0:37:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.149372:0:37:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787829985.149374:0:37:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787829985.149375:0:37:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787829985.149378:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff95dd87001000.
00000100:00000001:2.0:1787829985.149381:0:37:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829985.149385:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787829985.149392:0:37:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787829985.149394:0:37:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829985.149397:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084edf744] to pc [ptlrpcd_rcv+-1]  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787829985.149405:0:37:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829985.149409:0:37:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (4142)
00000100:00000001:1.0:1787829985.149427:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829985.149433:0:2346:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787829985.149436:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829985.149443:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149453:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829985.149455:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.149459:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829985.149461:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829985.149464:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829985.149468:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327398784:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829985.149472:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829985.149476:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829985.149479:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.149482:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829985.149487:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829985.149489:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829985.149495:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829985.149499:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829985.149505:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829985.149511:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398784, portal 25
00000100:00000001:1.0:1787829985.149514:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829985.149515:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402430400 : -116696307121216 : ffff95dd861143c0)
00000100:00000040:1.0:1787829985.149522:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149529:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829985.149532:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829985.149534:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829985.149538:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874671327398784, offset 0
00000400:00000200:1.0:1787829985.149559:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829985.149567:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.149572:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829985.149577:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829985.149581:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829985.149584:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829985.149587:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829985.149590:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40887:40887:256:4294967295] 192.168.203.116@tcp LPNI seq info [40887:40887:8:4294967295]
00000400:00000200:1.0:1787829985.149594:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.149602:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829985.149608:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.149613:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829985.149619:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829985.149623:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787829985.149627:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829985.149630:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787829985.149634:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.149636:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829985.149642:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829985.149646:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829985.149651:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829985.149653:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829985.149683:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829985.149688:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149696:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.149700:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149707:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.149709:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829985.149714:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829985.149718:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829985.149721:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 2 is 4298805933
00000100:00000010:1.0:1787829985.149726:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e63360.
00000100:00000001:1.0:1787829985.149732:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787829985.149737:0:2346:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS new deadline estimation for cpu 3 is 4298805933
00000100:00000010:1.0:1787829985.149739:0:2346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff95dd82e638a0.
00000100:00000001:1.0:1787829985.149741:0:2346:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.116@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787829985.149743:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829985.149745:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.149746:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829985.149750:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829985/real 1787829985]  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829985.149757:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829985.149758:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829985.149759:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829985.149761:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829985.149762:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829985.149764:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829985.149766:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829985.149771:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149775:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829985.149776:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.149777:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.149778:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829985.149780:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149785:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829985.149788:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149792:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829985.149793:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.149794:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.149795:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.149797:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829985.149798:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829985.149799:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd82e32000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829985.149802:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829985.149803:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829985.149805:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829985.149810:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:81801ac7-a7ac-4663-9aee-7c551b573152:2346:1874671327398784:192.168.203.116@tcp:250:kworker.0
00000100:00000001:1.0:1787829985.149813:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829985.149814:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149818:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829985.149819:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.149821:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829985.149823:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829985.149824:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829985.149827:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398912:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829985.149830:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829985.149832:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829985.149833:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.149835:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829985.149839:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829985.149840:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829985.149844:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829985.149846:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829985.149849:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829985.149852:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327398912, portal 10
00000100:00000001:1.0:1787829985.149854:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829985.149855:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402440000 : -116696307111616 : ffff95dd86116940)
00000100:00000040:1.0:1787829985.149862:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149866:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829985.149868:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829985.149870:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829985.149871:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874671327398912, offset 0
00000400:00000200:1.0:1787829985.149874:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829985.149877:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.149879:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829985.149881:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829985.149882:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829985.149884:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829985.149885:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829985.149887:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40888:40888:256:4294967295] 192.168.203.116@tcp LPNI seq info [40888:40888:8:4294967295]
00000400:00000200:1.0:1787829985.149890:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.149893:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829985.149896:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.149898:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829985.149900:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829985.149902:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787829985.149905:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829985.149906:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787829985.149909:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.149910:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829985.149912:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829985.149914:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829985.149916:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829985.149917:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829985.149918:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829985.149920:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149925:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.149927:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149931:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.149932:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829985.149933:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829985.149936:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: using a timeout of 9003 usecs (10 jiffies)
00000400:00000200:1.0:1787829985.149937:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829985.149938:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298805259
00000100:00000001:1.0:1787829985.149940:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298805933
00000100:00000001:1.0:1787829985.149941:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298805259
00000100:00000001:1.0:1787829985.149943:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298805933
00000100:00000001:1.0:1787829985.149944:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829985.149946:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.149948:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829985.149951:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829985/real 1787829985]  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829985.149958:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829985.149959:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829985.149961:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829985.149963:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829985.149964:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829985.149967:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829985.149971:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829985.149978:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149984:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829985.149985:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.149987:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.149988:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829985.149992:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.149998:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829985.150002:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150008:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829985.150010:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150012:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.150013:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150014:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829985.150016:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829985.150018:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd9273d000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829985.150022:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829985.150023:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829985.150027:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829985.150034:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86116940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327398912:192.168.203.116@tcp:38:kworker.0
00000100:00000001:1.0:1787829985.150038:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829985.150049:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150062:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829985.150063:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.150066:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829985.150067:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829985.150069:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829985.150072:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327399040:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829985.150075:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829985.150077:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829985.150079:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.150080:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829985.150083:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829985.150086:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829985.150089:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829985.150092:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829985.150095:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829985.150098:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327399040, portal 4
00000100:00000001:1.0:1787829985.150099:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829985.150100:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402434240 : -116696307117376 : ffff95dd861152c0)
00000100:00000040:1.0:1787829985.150106:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150112:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829985.150114:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829985.150116:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829985.150118:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327399040, offset 0
00000400:00000200:1.0:1787829985.150122:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829985.150126:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.150129:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829985.150132:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829985.150134:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829985.150136:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829985.150138:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829985.150141:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40889:40889:256:4294967295] 192.168.203.116@tcp LPNI seq info [40889:40889:8:4294967295]
00000400:00000200:1.0:1787829985.150144:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.150162:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829985.150167:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.150173:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829985.150177:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829985.150180:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787829985.150183:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829985.150186:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787829985.150189:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.150191:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829985.150194:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829985.150197:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829985.150200:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829985.150202:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829985.150204:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829985.150208:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150215:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.150218:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150224:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.150225:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829985.150228:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829985.150231:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: using a timeout of 16897 usecs (17 jiffies)
00000400:00000200:1.0:1787829985.150233:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829985.150236:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298805266
00000100:00000001:1.0:1787829985.150238:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298805933
00000100:00000001:1.0:1787829985.150240:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298805266
00000100:00000001:1.0:1787829985.150242:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298805933
00000100:00000001:1.0:1787829985.150244:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829985.150246:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.150248:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829985.150251:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829985/real 1787829985]  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829985.150256:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829985.150257:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829985.150258:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829985.150259:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829985.150260:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829985.150262:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829985.150264:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829985.150268:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150272:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829985.150272:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150273:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.150274:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829985.150276:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150280:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829985.150282:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150286:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829985.150287:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150288:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.150289:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150290:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829985.150291:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829985.150292:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd90720800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829985.150294:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829985.150295:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829985.150298:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829985.150302:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd861152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327399040:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829985.150304:0:2346:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787829985.150306:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150309:0:2346:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787829985.150310:0:2346:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.150311:0:2346:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787829985.150312:0:2346:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787829985.150313:0:2346:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787829985.150315:0:2346:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327399168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829985.150317:0:2346:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787829985.150325:0:2346:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787829985.150327:0:2346:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787829985.150328:0:2346:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787829985.150331:0:2346:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829985.150333:0:2346:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787829985.150336:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000010:1.0:1787829985.150338:0:2346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff95dd94630738.
00000400:00000010:1.0:1787829985.150340:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98220.
00000100:00000200:1.0:1787829985.150342:0:2346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874671327399168, portal 4
00000100:00000001:1.0:1787829985.150344:0:2346:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787829985.150345:0:2346:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446627377402431360 : -116696307120256 : ffff95dd86114780)
00000100:00000040:1.0:1787829985.150351:0:2346:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150357:0:2346:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787829985.150359:0:2346:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.116@tcp
00000400:00000010:1.0:1787829985.150361:0:2346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff95dd84f98bb0.
00000100:00000200:1.0:1787829985.150364:0:2346:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874671327399168, offset 0
00000400:00000200:1.0:1787829985.150368:0:2346:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.116@tcp
00000400:00000200:1.0:1787829985.150372:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.150375:0:2346:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.116@tcp local destination
00000400:00000200:1.0:1787829985.150377:0:2346:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787829985.150379:0:2346:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.16@tcp
00000400:00000200:1.0:1787829985.150381:0:2346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787829985.150383:0:2346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.116@tcp
00000400:00000200:1.0:1787829985.150386:0:2346:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.16@tcp NI seq info: [40890:40890:256:4294967295] 192.168.203.116@tcp LPNI seq info [40890:40890:8:4294967295]
00000400:00000200:1.0:1787829985.150389:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000400:00000200:1.0:1787829985.150393:0:2346:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.16@tcp(192.168.203.16@tcp:<?>) -> 192.168.203.116@tcp(192.168.203.116@tcp:192.168.203.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787829985.150396:0:2346:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.150398:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff95dd99955f00.
00000800:00000200:1.0:1787829985.150401:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.0:1787829985.150403:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787829985.150407:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff95dd9271d0c0] -> 12345-192.168.203.116@tcp (2)
00000800:00000200:1.1:1787829985.150409:0:2346:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.116 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787829985.150425:0:2346:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.116@tcp
00000800:00000010:1.0:1787829985.150427:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff95dd99955f00.
00000400:00000200:1.0:1787829985.150429:0:2346:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787829985.150432:0:2346:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.16@tcp->192.168.203.116@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787829985.150436:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98bb0
00000400:00000010:1.0:1787829985.150437:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98bb0.
00000100:00000001:1.0:1787829985.150440:0:2346:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787829985.150443:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150451:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.150454:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 0 dl 1787830010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150458:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.150459:0:2346:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787829985.150462:0:2346:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787829985.150464:0:2346:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: using a timeout of 21158 usecs (22 jiffies)
00000400:00000200:1.0:1787829985.150466:0:2346:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.116@tcp to cpt 1
00000100:00000001:1.0:1787829985.150467:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 2 is 4298805271
00000100:00000001:1.0:1787829985.150469:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 2, keep current deadline 4298805933
00000100:00000001:1.0:1787829985.150470:0:2346:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS new deadline estimation for cpu 3 is 4298805271
00000100:00000001:1.0:1787829985.150471:0:2346:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff95dd94719800: PM QoS request active for cpu 3, keep current deadline 4298805933
00000100:00000001:1.0:1787829985.150472:0:2346:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787829985.150474:0:2346:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.150475:0:2346:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787829985.150477:0:2346:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829985/real 1787829985]  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787829985.150481:0:2346:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787829985.150482:0:2346:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787829985.150483:0:2346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff95dd94630738.
00000400:00000200:1.0:1787829985.150484:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff95dd84f98220
00000400:00000010:1.0:1787829985.150485:0:2346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff95dd84f98220.
00000100:00000001:1.0:1787829985.150486:0:2346:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787829985.150488:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787829985.150492:0:2346:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150496:0:2346:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787829985.150496:0:2346:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150498:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.150498:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787829985.150500:0:2346:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150504:0:2346:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787829985.150506:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150510:0:2346:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787829985.150512:0:2346:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150513:0:2346:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787829985.150513:0:2346:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150514:0:2346:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787829985.150516:0:2346:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787829985.150517:0:2346:0:(import.c:65:import_set_state_nolock()) ffff95dd91f74800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787829985.150519:0:2346:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.116@tcp failed (-110)
00000100:00000001:1.0:1787829985.150520:0:2346:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787829985.150522:0:2346:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787829985.150526:0:2346:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff95dd86114780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:487ec2ab-0772-4b76-b5b3-7e1d3e9f5308:2346:1874671327399168:192.168.203.116@tcp:8:kworker.0
00000100:00000001:1.0:1787829985.150528:0:2346:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150529:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.150532:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861143c0 x1874671327398784/t0(0) o250->MGC192.168.203.116@tcp@192.168.203.116@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150538:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829985.150540:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829985.150552:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f7400.
02000000:00000001:1.0:1787829985.150555:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829985.150557:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829985.150559:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd82e32000 refcount=3 obd=MGC192.168.203.116@tcp
00000020:00000001:1.0:1787829985.150561:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829985.150563:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001800.
02000000:00000001:1.0:1787829985.150567:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829985.150569:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829985.150570:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861143c0.
00000100:00000001:1.0:1787829985.150573:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829985.150574:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150576:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.150580:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86116940 x1874671327398912/t0(0) o38->lustre-MDT0000-mdc-ffff95dd94719800@192.168.203.116@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150587:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829985.150588:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829985.150589:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6800.
02000000:00000001:1.0:1787829985.150592:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829985.150593:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829985.150594:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd9273d000 refcount=71 obd=lustre-MDT0000-mdc-ffff95dd94719800
00000020:00000001:1.0:1787829985.150596:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829985.150598:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87003000.
02000000:00000001:1.0:1787829985.150600:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829985.150601:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829985.150602:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86116940.
00000100:00000001:1.0:1787829985.150604:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829985.150605:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150607:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.150610:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd861152c0 x1874671327399040/t0(0) o8->lustre-OST0000-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150615:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829985.150616:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829985.150618:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f6400.
02000000:00000001:1.0:1787829985.150620:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829985.150621:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829985.150622:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd90720800 refcount=2 obd=lustre-OST0000-osc-ffff95dd94719800
00000020:00000001:1.0:1787829985.150624:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829985.150625:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87002000.
02000000:00000001:1.0:1787829985.150627:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829985.150628:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829985.150630:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd861152c0.
00000100:00000001:1.0:1787829985.150631:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829985.150632:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150634:0:2346:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787829985.150637:0:2346:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff95dd86114780 x1874671327399168/t0(0) o8->lustre-OST0001-osc-ffff95dd94719800@192.168.203.116@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787829985.150648:0:2346:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787829985.150649:0:2346:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787829985.150651:0:2346:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff95dd860f5400.
02000000:00000001:1.0:1787829985.150653:0:2346:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787829985.150653:0:2346:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787829985.150654:0:2346:0:(genops.c:1223:class_import_put()) import ffff95dd91f74800 refcount=3 obd=lustre-OST0001-osc-ffff95dd94719800
00000020:00000001:1.0:1787829985.150670:0:2346:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787829985.150671:0:2346:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff95dd87001000.
02000000:00000001:1.0:1787829985.150674:0:2346:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787829985.150675:0:2346:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787829985.150676:0:2346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff95dd86114780.
00000100:00000001:1.0:1787829985.150678:0:2346:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787829985.150679:0:2346:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150681:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787829985.150688:0:2346:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787829985.150690:0:2346:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787829985.150693:0:2346:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787829985.150694:0:2346:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787829985.660766:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829985.660772:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829985.660773:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829985.660777:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829985.660779:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829985.660782:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829985.660784:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829985.660786:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829985.660788:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787829985.852764:0:16534:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfaa20 (cpu 2) has reached its deadline 4298805933, at 4298805952
00000100:00000040:3.0:1787829985.852764:0:16511:0:(connection.c:75:cpu_latency_work()) work item of ffff95dd82cfab30 (cpu 3) has reached its deadline 4298805933, at 4298805952
00000100:00000010:2.0:1787829985.852777:0:16534:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e63360.
00000100:00000010:3.0:1787829985.852779:0:16511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff95dd82e638a0.
00010000:00000001:0.0:1787829985.981871:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829985.981879:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829985.981882:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829985.981885:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981890:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829985.981892:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981895:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981897:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981901:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829985.981903:0:16513:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:0.0:1787829985.981905:0:16513:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:0.0:1787829985.981906:0:16513:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981908:0:16513:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:0.0:1787829985.981909:0:16513:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981911:0:16513:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981913:0:16513:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829985.981917:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829985.981918:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829985.982049:0:5589:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829985.982052:0:5589:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829985.982055:0:5589:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829986.684733:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829986.684740:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829986.684743:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829986.684748:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829986.684751:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829986.684755:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829986.684758:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829986.684760:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829986.684763:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829987.004762:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829987.004768:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829987.004771:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829987.004772:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829987.004774:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829987.004775:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787829987.004830:0:5588:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787829987.004834:0:5588:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787829987.004838:0:5588:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787829987.708888:0:16006:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787829987.708895:0:16006:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787829987.708898:0:16006:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787829987.708902:0:16006:0:(client.c:2715:ptlrpc_set_wait()) set ffff95dd87048f00 going to sleep for 0 seconds
00000100:00000001:2.0:1787829987.708906:0:16006:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787829987.708909:0:16006:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829987.708912:0:16006:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787829987.708915:0:16006:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787829987.708919:0:16006:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829988.029029:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829988.029036:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829988.029042:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829988.029043:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:0.0:1787829988.029046:0:16513:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:0.0:1787829988.029048:0:16513:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787829988.029064:0:6051:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787829988.029067:0:6051:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787829988.029069:0:6051:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
Debug log: 128566 lines, 128566 kept, 0 dropped, 0 bad.